hub v0.91.0 — the staleness window learns each tier's own rhythm (R-86 Part 2)

Ships WITH agent v0.121.0, not after it. The agent now proves a tier once per ARCHIVE GENERATION, so a weekly tier is proved weekly — in perfect health. The flat 7-day restoreProvenStaleAfter derived its number from the 24h cadence R-86 removes, and a healthy weekly tier's proof age reaches EXACTLY 168h just before its next proof: it sat ON the line, so any ordinary delay tipped it into a nightly alarm about a working system. restoreProvenWindow(tier, observed, ok): - the tier's own archive interval, OBSERVED from reports the hub already holds (pbs_snapshots + successful backups attributed by TARGET TYPE, slice A.4) - x4 generations = the same tolerance the flat constant expressed - floored at 7d (never tighter than before), capped at 12d (strictly inside the 2-week offsite retention) - falls back to the DECLARED rhythm (26h host / 8d offsite — the thresholds the backup-freshness checker already uses) when history is too short to observe one; falling back to the FLOOR would recreate the false alarm on a fresh box Kept: absence is UNKNOWN until the anchored window passes; the signal stays edge-triggered; failed and stale remain distinct events. Every reason string now states the window it was judged against (R-100's corollary). Also backfills the missing v0.90.1 CHANGELOG entry (deployed since f21e7ca), and records the operator's 2026-08-03 ruling that ep0 is Tier 2 / protected.
2026-08-03 15:03:35 +02:00
parent e34b614e5b
commit 323f45a5ef
4 changed files with 478 additions and 41 deletions
@@ -182,26 +182,33 @@ func TestRestoreTest_NewbornDoesNotAlarm(t *testing.T) {
 	}
 }

-// The boundary, pinned by name so a refactor has to delete an obviously-named contract.
+// The boundary, pinned by name so a refactor has to delete an obviously-named contract. R-86 made
+// the limit per-tier, so the anchor is now measured against THE TIER'S OWN window — here the local
+// tier's, which clamps to the 7-day floor and so keeps this contract numerically identical to the
+// one the flat constant expressed.
 func TestRestoreTest_Contract_UnprovenIsUnknownUntilTheAnchorElapses(t *testing.T) {
 	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)
+	window := restoreProvenWindow("local", 24*time.Hour, true)
+	if window != restoreProvenWindowFloor {
+		t.Fatalf("precondition: a daily local tier must clamp to the floor; got %s", window)
+	}
 	cases := []struct {
 		name       string
 		watched    time.Duration
 		wantMissed bool
 	}{
 		{"newborn, 1h", time.Hour, false},
-		{"just inside", restoreProvenStaleAfter - time.Minute, false},
-		{"exactly at the limit", restoreProvenStaleAfter, false},
-		{"just outside", restoreProvenStaleAfter + time.Minute, true},
+		{"just inside", window - time.Minute, false},
+		{"exactly at the limit", window, false},
+		{"just outside", window + time.Minute, true},
 		{"long past", 30 * 24 * time.Hour, true},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			got := assessRestoreProven("pbs", time.Time{}, now.Add(-c.watched), now)
+			got := assessRestoreProven("local", time.Time{}, now.Add(-c.watched), now, window)
 			if got.missed() != c.wantMissed {
 				t.Fatalf("CONTRACT VIOLATED: unproven for %s (limit %s) → missed=%v, want %v (reason %q)",
-					c.watched, restoreProvenStaleAfter, got.missed(), c.wantMissed, got.reason)
+					c.watched, window, got.missed(), c.wantMissed, got.reason)
 			}
 			if !c.wantMissed && got.verdict != verdictUnknown {
 				t.Fatalf("a deferred tier must be UNKNOWN (visible), not OK; got verdict=%d", got.verdict)
@@ -215,15 +222,17 @@ func TestRestoreTest_Contract_UnprovenIsUnknownUntilTheAnchorElapses(t *testing.
 func TestRestoreTest_StaleIsSeparateFromFailure(t *testing.T) {
 	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)

-	stale := assessRestoreProven("pbs", now.Add(-9*24*time.Hour), now.Add(-60*24*time.Hour), now)
+	// A DAILY tier judged on its own rhythm: the window clamps to the 7-day floor.
+	daily := restoreProvenWindow("local", 24*time.Hour, true)
+	stale := assessRestoreProven("local", now.Add(-9*24*time.Hour), now.Add(-60*24*time.Hour), now, daily)
 	if !stale.missed() {
-		t.Fatalf("a tier last proven 9 days ago (limit %s) must be stale; got %q", restoreProvenStaleAfter, stale.reason)
+		t.Fatalf("a daily tier last proven 9 days ago (limit %s) must be stale; got %q", daily, stale.reason)
 	}
 	// The wording must not read as "broken" — that is the other signal.
 	if !strings.Contains(stale.reason, "unverified, not known-broken") {
 		t.Fatalf("staleness must say UNVERIFIED, not broken — merging the two is the thing this avoids; got %q", stale.reason)
 	}
-	fresh := assessRestoreProven("pbs", now.Add(-2*24*time.Hour), now.Add(-60*24*time.Hour), now)
+	fresh := assessRestoreProven("local", now.Add(-2*24*time.Hour), now.Add(-60*24*time.Hour), now, daily)
 	if fresh.verdict != verdictOK {
 		t.Fatalf("a tier proven 2 days ago is fine; got verdict=%d reason=%q", fresh.verdict, fresh.reason)
 	}
@@ -283,3 +292,199 @@ func boolStr(b bool) string {
 	}
 	return "false"
 }
+
+// ── SCENARIO G — a healthy WEEKLY tier is never reported stale (R-86 Part 2) ─────────────────
+//
+// This is the test that pins the false alarm this change would otherwise have CREATED. The agent
+// now proves a tier once per archive generation, so a weekly offsite tier is proved weekly — in
+// perfect health. Against the old flat 7-day window it would sit on the line and alarm every night.
+//
+// COMPANION RED-PROOF (observed 2026-08-03): pin the window flat, as it was —
+//
+//   - window := restoreProvenWindow(tier, observed, observedOK)
+//   - window := restoreProvenWindowFloor   // the pre-R-86 flat 7 days
+//
+// →	--- FAIL: TestRestoreTest_HealthyWeeklyTierIsNeverStale
+//
+//	week 0: a weekly tier proved on its own archive must never be stale (proof age 172h0m0s,
+//	window 168h0m0s); verdict=2 reason="pbs tier: last successful restore-test was 172h0m0s ago
+//	(limit 168h0m0s, this tier's own backup rhythm) — the tier is unverified, not known-broken"
+//
+// Restored. The mutation is one line because the whole of Part 2 is one decision: whose rhythm.
+//
+// NOTE, because it is the finding this test nearly hid: the FIRST version of this fixture had NO
+// jitter, and it PASSED under the mutation. A perfectly regular weekly tier's proof age reaches
+// EXACTLY 168h just before the next proof, and `age > window` is false by a hair — a hollow test
+// that would have shipped Part 1 and its false alarm together. The jitter below is what makes this
+// a test, and it is also the truth about the old constant: a healthy weekly tier did not merely sit
+// near the line, it sat ON it, so any ordinary delay tipped it over.
+func TestRestoreTest_HealthyWeeklyTierIsNeverStale(t *testing.T) {
+	start := time.Date(2026, 6, 1, 3, 0, 0, 0, time.UTC)
+	firstContact := start.Add(-24 * time.Hour)
+
+	// The observable rhythm of a weekly tier, as the hub would compute it from the reports. No
+	// assertion about the window ITSELF here on purpose: that is the mechanism, and it is pinned in
+	// TestRestoreProvenWindow_Contract. What this test asserts is the CONSEQUENCE — does the alarm
+	// fire? — because R-97b proved a mechanism and shipped a broken consequence anyway.
+	weekly := restoreProvenWindow("pbs", 7*24*time.Hour, true)
+
+	// Walk several weeks of a HEALTHY tier, with the jitter a real one has: the backup does not land
+	// to the second, and a restore-test can be deferred one evaluation behind a running backup.
+	//
+	// The jitter is the point. A perfectly regular weekly tier's proof reaches an age of EXACTLY one
+	// interval (168h) just before the next proof, and against a flat 168h window `age > window` is
+	// false by a hair — so a regular fixture would pass against the very constant this task must
+	// change, and prove nothing. That is the brief's "sits exactly on that line": every real-world
+	// delay pushes it over, and the alarm is about a system that is working.
+	settle, evalLatency := 24*time.Hour, 6*time.Hour
+	archiveLate := []time.Duration{0, 4 * time.Hour, 2 * time.Hour, 6 * time.Hour, 0, 3 * time.Hour}
+	deferred := []time.Duration{0, 0, 6 * time.Hour, 0, 0, 6 * time.Hour} // one evaluation behind a backup
+
+	archiveAt := func(week int) time.Time {
+		return start.AddDate(0, 0, 7*week).Add(archiveLate[week])
+	}
+	provenAt := func(week int) time.Time {
+		return archiveAt(week).Add(settle + evalLatency + deferred[week])
+	}
+
+	var worst time.Duration
+	for week := 0; week+1 < len(archiveLate); week++ {
+		// The widest the proof's age ever gets: the instant before the NEXT week's proof lands.
+		now := provenAt(week + 1).Add(-time.Second)
+		age := now.Sub(provenAt(week))
+		if age > worst {
+			worst = age
+		}
+		v := assessRestoreProven("pbs", provenAt(week), firstContact, now, weekly)
+		if v.verdict != verdictOK {
+			t.Fatalf("week %d: a weekly tier proved on its own archive must never be stale (proof age %s, window %s); verdict=%d reason=%q",
+				week, age.Round(time.Hour), weekly, v.verdict, v.reason)
+		}
+	}
+	// The fixture must actually EXERCISE the boundary — a jitter-free walk would sit at exactly one
+	// interval and pass against a flat 7-day window, which is the hollow version of this test.
+	if worst <= restoreProvenWindowFloor {
+		t.Fatalf("this fixture never exceeds the old flat window (worst proof age %s) — it cannot detect the defect it exists for", worst)
+	}
+}
+
+// ...and a weekly tier that genuinely STOPS being proved must still alarm. A window that never
+// fires is not a fix, it is a deletion.
+func TestRestoreTest_WeeklyTierThatStopsBeingProvedStillAlarms(t *testing.T) {
+	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)
+	weekly := restoreProvenWindow("pbs", 7*24*time.Hour, true)
+
+	v := assessRestoreProven("pbs", now.Add(-weekly-time.Hour), now.Add(-90*24*time.Hour), now, weekly)
+	if !v.missed() {
+		t.Fatalf("a weekly tier unproven for longer than its own window MUST alarm; got verdict=%d reason=%q", v.verdict, v.reason)
+	}
+	if !strings.Contains(v.reason, weekly.String()) {
+		t.Fatalf("the alarm must state the window it was judged against (R-100's corollary); got %q", v.reason)
+	}
+}
+
+// The window's own contract: derived from the tier's rhythm, floored, capped, and never dependent
+// on an unobservable history for the tier that would suffer most from a wrong answer.
+func TestRestoreProvenWindow_Contract(t *testing.T) {
+	cases := []struct {
+		name       string
+		tier       string
+		observed   time.Duration
+		observedOK bool
+		want       time.Duration
+	}{
+		{"daily local clamps to the floor", "local", 24 * time.Hour, true, restoreProvenWindowFloor},
+		{"weekly pbs widens", "pbs", 7 * 24 * time.Hour, true, restoreProvenWindowCap},
+		{"3-day tier sits between", "pbs", 72 * time.Hour, true, 12 * 24 * time.Hour},
+		{"unobservable local falls back to its declared rhythm", "local", 0, false, restoreProvenWindowFloor},
+		{"unobservable pbs falls back WIDE, not to the floor", "pbs", 0, false, restoreProvenWindowCap},
+		{"a nonsense zero interval is ignored", "pbs", 0, true, restoreProvenWindowCap},
+	}
+	// The relationship Part 1 depends on: a weekly tier's window must be WIDER than a daily tier's,
+	// or proving weekly (which is now correct behaviour) alarms on itself.
+	if restoreProvenWindow("pbs", 7*24*time.Hour, true) <= restoreProvenWindow("local", 24*time.Hour, true) {
+		t.Fatal("a weekly tier must earn a wider window than a daily one — otherwise R-86's agent half alarms about itself")
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := restoreProvenWindow(c.tier, c.observed, c.observedOK)
+			if got != c.want {
+				t.Fatalf("window(%s, observed=%s ok=%v) = %s, want %s", c.tier, c.observed, c.observedOK, got, c.want)
+			}
+			if got < restoreProvenWindowFloor || got > restoreProvenWindowCap {
+				t.Fatalf("every window must stay inside [%s, %s]; got %s", restoreProvenWindowFloor, restoreProvenWindowCap, got)
+			}
+		})
+	}
+}
+
+// The rhythm must be OBSERVED from the reports, not assumed — including the slice-A.4 rule that a
+// PBS-targeted vzdump appears in both arrays and must be attributed by TARGET TYPE.
+func TestObservedArchiveIntervals_FromReports(t *testing.T) {
+	base := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC)
+	mk := func(localAt []time.Time, pbsAt []time.Time) string {
+		type stg struct{ Name, Type, Content string }
+		type bk struct {
+			TargetID  string `json:"target_id"`
+			Success   bool   `json:"success"`
+			StartedAt string `json:"started_at"`
+		}
+		type snap struct {
+			BackupTime string `json:"backup_time"`
+		}
+		payload := struct {
+			StorageTargets []struct {
+				Name    string `json:"name"`
+				Type    string `json:"type"`
+				Content string `json:"content"`
+			} `json:"storage_targets"`
+			Backups      []bk   `json:"backups"`
+			PBSSnapshots []snap `json:"pbs_snapshots"`
+		}{}
+		payload.StorageTargets = append(payload.StorageTargets, struct {
+			Name    string `json:"name"`
+			Type    string `json:"type"`
+			Content string `json:"content"`
+		}{"felhom-backup", "dir", "backup"}, struct {
+			Name    string `json:"name"`
+			Type    string `json:"type"`
+			Content string `json:"content"`
+		}{"felhom-pbs", "pbs", "backup"})
+		for _, at := range localAt {
+			payload.Backups = append(payload.Backups, bk{"felhom-backup", true, at.Format(time.RFC3339)})
+		}
+		for _, at := range pbsAt {
+			// The SAME archive appears as a vzdump record AND as a snapshot — slice A.4.
+			payload.Backups = append(payload.Backups, bk{"felhom-pbs", true, at.Format(time.RFC3339)})
+			payload.PBSSnapshots = append(payload.PBSSnapshots, snap{at.Format(time.RFC3339)})
+		}
+		b, err := json.Marshal(payload)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return string(b)
+	}
+
+	rows := []store.HostReportRow{
+		{ReportJSON: mk(
+			[]time.Time{base, base.AddDate(0, 0, 1), base.AddDate(0, 0, 2)},
+			[]time.Time{base, base.AddDate(0, 0, 7)},
+		)},
+		{ReportJSON: `{{{malformed`}, // must not blind the scan
+	}
+
+	got := observedArchiveIntervals(rows)
+	if d, ok := got["local"]; !ok || d != 24*time.Hour {
+		t.Fatalf("a daily host tier must be observed as ~24h; got %s ok=%v", d, ok)
+	}
+	if d, ok := got["pbs"]; !ok || d != 7*24*time.Hour {
+		t.Fatalf("a weekly offsite tier must be observed as ~7d — and its vzdump record must NOT be "+
+			"counted into the host tier (slice A.4); got %s ok=%v", d, ok)
+	}
+
+	// One generation is not a rhythm: unobservable, so the caller falls back to the declared one.
+	single := []store.HostReportRow{{ReportJSON: mk(nil, []time.Time{base})}}
+	if d, ok := observedArchiveIntervals(single)["pbs"]; ok {
+		t.Fatalf("one archive cannot yield an interval; got %s", d)
+	}
+}