R-86: restore-test follows the backup, not the clock (v0.121.0)

The ticker survives as the EVALUATION interval only. A tier is DUE when its
newest archive that has settled for `settle` (default 24h) has not been proven:
daily tier -> proved daily on yesterday's archive, weekly tier -> weekly on its
own, newborn -> UNKNOWN.

The trap avoided: the literal reading ("newest archive is >= 24h old") is NEVER
true on a daily tier, so it silently switches restore-testing off where it
matters most. Red-proved at 0 runs over 5 simulated days.

- state records WHICH archive was proven; legacy files keep their time and yield
  no proven archive (each tier due once after the upgrade, deliberately)
- two knobs replace one: restore_test_eval_interval_seconds (6h, measured) and
  restore_test_settle_seconds (24h). The old cadence key keeps its DISABLE
  meaning verbatim and now seeds the settle lag, with a start-up WARN.
- due-check runs BEFORE the heavy-op gate (a frequent poll must not make a
  starting backup record a failure, F-A1)
- candidate picker skips implausible archives (a phantom would be due forever)
- new read-only --selftest=restore-test-due prints the verdict + its cost

internal/backup/restoretest_state.go

@@ -28,42 +28,92 @@ import (
 //
 // Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time
 // would look freshly proven and stop being retried — the same "a failure satisfies the cadence"
-// trap the backup due-check avoids.
+// trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job:
+// the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from
+// ever becoming due again. The rule earns its keep twice now.
+//
+// R-86 (1.2) — WHICH ARCHIVE, not just when.
+//
+// A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is
+// consistent both with "yesterday's archive is proven" and with "an archive from a week ago is
+// proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION —
+// a tier is due once it holds a settled archive that has not been proven — so the identity of the
+// proven archive is the state, and the time is metadata (rotation ordering, operator reporting).
+//
+// This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT":
+// here it records a result, but not WHICH result, and that is just as unable to answer the question
+// being asked of it.
 type RestoreTestState struct {
 	path string
 	mu   sync.Mutex
-	last map[string]time.Time // target id → last SUCCESSFUL restore-test (UTC)
+	last map[string]provenTier // target id → what was last PROVEN on that tier
+}
+
+// provenTier is one tier's proof: the archive that passed, and when it passed.
+type provenTier struct {
+	Archive string    // volid of the archive that PASSED; "" = a legacy record with no archive
+	At      time.Time // when that run passed (UTC)
+}
+
+// provenTierJSON is the on-disk shape (R-86). The legacy shape was a bare RFC3339 STRING per
+// target; both are read, only this one is written — see NewRestoreTestState.
+type provenTierJSON struct {
+	Archive  string `json:"archive"`
+	ProvenAt string `json:"proven_at"`
 }
 
 // NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an
 // error: it degrades to "nothing proven yet", which is the correct starting point and keeps a
 // corrupt file from wedging the daemon.
+//
+// MIGRATION (R-86). The pre-R-86 file is `{"<target>": "<RFC3339>"}` — a time and no archive. A
+// legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists
+// at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after
+// the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to
+// read a legacy time as proof of whatever archive happens to be current, which would mark an
+// unproven archive proven — inventing a guarantee out of a migration.
 func NewRestoreTestState(path string) *RestoreTestState {
-	s := &RestoreTestState{path: path, last: map[string]time.Time{}}
+	s := &RestoreTestState{path: path, last: map[string]provenTier{}}
 	data, err := os.ReadFile(path)
 	if err != nil {
 		return s
 	}
-	var raw map[string]string
+	var raw map[string]json.RawMessage
 	if json.Unmarshal(data, &raw) != nil {
 		return s
 	}
-	for target, ts := range raw {
-		if t, perr := time.Parse(time.RFC3339, ts); perr == nil {
-			s.last[target] = t.UTC()
+	for target, msg := range raw {
+		// Legacy shape: a bare RFC3339 string.
+		var legacy string
+		if json.Unmarshal(msg, &legacy) == nil {
+			if t, perr := time.Parse(time.RFC3339, legacy); perr == nil {
+				s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately
+			}
+			continue
 		}
+		var cur provenTierJSON
+		if json.Unmarshal(msg, &cur) != nil {
+			continue // one unreadable entry must not lose the others
+		}
+		t, perr := time.Parse(time.RFC3339, cur.ProvenAt)
+		if perr != nil {
+			continue
+		}
+		s.last[target] = provenTier{Archive: cur.Archive, At: t.UTC()}
 	}
 	return s
 }
 
-// RecordSuccess stamps a tier as proven at t. Only call this for a PASSING restore-test.
-func (s *RestoreTestState) RecordSuccess(target string, t time.Time) error {
+// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed. Only call this for a
+// PASSING restore-test — the archive is what makes the tier not-due, so recording one for a failed
+// run would retire the archive unproven.
+func (s *RestoreTestState) RecordSuccess(target, archive string, t time.Time) error {
 	if target == "" {
 		return nil
 	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.last[target] = t.UTC()
+	s.last[target] = provenTier{Archive: archive, At: t.UTC()}
 	return s.saveLocked()
 }
 
@@ -71,17 +121,30 @@ func (s *RestoreTestState) RecordSuccess(target string, t time.Time) error {
 func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	t, ok := s.last[target]
-	return t, ok
+	p, ok := s.last[target]
+	return p.At, ok
 }
 
-// Snapshot returns a copy of the whole map — for the host-report gauge.
+// ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested,
+// or a legacy record carrying only a time). It is the due-check's whole question: an archive that is
+// not this one has not been proven.
+func (s *RestoreTestState) ProvenArchive(target string) (string, bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, ok := s.last[target]
+	if !ok || p.Archive == "" {
+		return "", false
+	}
+	return p.Archive, true
+}
+
+// Snapshot returns a copy of the last-proven TIMES — for the host-report gauge.
 func (s *RestoreTestState) Snapshot() map[string]time.Time {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	out := make(map[string]time.Time, len(s.last))
 	for k, v := range s.last {
-		out[k] = v
+		out[k] = v.At
 	}
 	return out
 }
@@ -100,8 +163,9 @@ func (s *RestoreTestState) OldestFirst(targets []string) []string {
 	defer s.mu.Unlock()
 	out := append([]string(nil), targets...)
 	sort.SliceStable(out, func(i, j int) bool {
-		ti, oki := s.last[out[i]]
-		tj, okj := s.last[out[j]]
+		pi, oki := s.last[out[i]]
+		pj, okj := s.last[out[j]]
+		ti, tj := pi.At, pj.At
 		switch {
 		case !oki && !okj:
 			return out[i] < out[j] // both never proven → deterministic
@@ -119,9 +183,9 @@ func (s *RestoreTestState) OldestFirst(targets []string) []string {
 }
 
 func (s *RestoreTestState) saveLocked() error {
-	raw := make(map[string]string, len(s.last))
-	for target, t := range s.last {
-		raw[target] = t.UTC().Format(time.RFC3339)
+	raw := make(map[string]provenTierJSON, len(s.last))
+	for target, p := range s.last {
+		raw[target] = provenTierJSON{Archive: p.Archive, ProvenAt: p.At.UTC().Format(time.RFC3339)}
 	}
 	data, err := json.MarshalIndent(raw, "", "  ")
 	if err != nil {