package backup import ( "encoding/json" "os" "path/filepath" "sort" "sync" "time" ) // RestoreTestState persists the last SUCCESSFUL restore-test per backup tier. // // R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth // stating, because the two look like the same problem and are not: // // - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage, // so the agent could ask "when did a backup last land?" and never persist anything. That is // strictly better, because a pruned archive correctly stops counting. // - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is // nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it // must be written down or it is lost. // // Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map // would reset every tier to "never tested" on each restart. Ordering would then depend on map // iteration order, and one tier could be starved indefinitely while the other is re-tested — with // agent deploys as routine as they are, that is not a corner case. // // Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time // would look freshly proven and stop being retried — the same "a failure satisfies the cadence" // trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job: // the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from // ever becoming due again. The rule earns its keep twice now. // // R-86 (1.2) — WHICH ARCHIVE, not just when. // // A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is // consistent both with "yesterday's archive is proven" and with "an archive from a week ago is // proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION — // a tier is due once it holds a settled archive that has not been proven — so the identity of the // proven archive is the state, and the time is metadata (rotation ordering, operator reporting). // // This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT": // here it records a result, but not WHICH result, and that is just as unable to answer the question // being asked of it. type RestoreTestState struct { path string mu sync.Mutex last map[string]provenTier // target id → what was last PROVEN on that tier } // provenTier is one tier's proof: the archive that passed, and when it passed. type provenTier struct { Archive string // volid of the archive that PASSED; "" = a legacy record with no archive At time.Time // when that run passed (UTC) } // provenTierJSON is the on-disk shape (R-86). The legacy shape was a bare RFC3339 STRING per // target; both are read, only this one is written — see NewRestoreTestState. type provenTierJSON struct { Archive string `json:"archive"` ProvenAt string `json:"proven_at"` } // NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an // error: it degrades to "nothing proven yet", which is the correct starting point and keeps a // corrupt file from wedging the daemon. // // MIGRATION (R-86). The pre-R-86 file is `{"": ""}` — a time and no archive. A // legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists // at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after // the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to // read a legacy time as proof of whatever archive happens to be current, which would mark an // unproven archive proven — inventing a guarantee out of a migration. func NewRestoreTestState(path string) *RestoreTestState { s := &RestoreTestState{path: path, last: map[string]provenTier{}} data, err := os.ReadFile(path) if err != nil { return s } var raw map[string]json.RawMessage if json.Unmarshal(data, &raw) != nil { return s } for target, msg := range raw { // Legacy shape: a bare RFC3339 string. var legacy string if json.Unmarshal(msg, &legacy) == nil { if t, perr := time.Parse(time.RFC3339, legacy); perr == nil { s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately } continue } var cur provenTierJSON if json.Unmarshal(msg, &cur) != nil { continue // one unreadable entry must not lose the others } t, perr := time.Parse(time.RFC3339, cur.ProvenAt) if perr != nil { continue } s.last[target] = provenTier{Archive: cur.Archive, At: t.UTC()} } return s } // RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed. Only call this for a // PASSING restore-test — the archive is what makes the tier not-due, so recording one for a failed // run would retire the archive unproven. func (s *RestoreTestState) RecordSuccess(target, archive string, t time.Time) error { if target == "" { return nil } s.mu.Lock() defer s.mu.Unlock() s.last[target] = provenTier{Archive: archive, At: t.UTC()} return s.saveLocked() } // LastSuccess returns when this tier was last proven (ok=false = never). func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) { s.mu.Lock() defer s.mu.Unlock() p, ok := s.last[target] return p.At, ok } // ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested, // or a legacy record carrying only a time). It is the due-check's whole question: an archive that is // not this one has not been proven. func (s *RestoreTestState) ProvenArchive(target string) (string, bool) { s.mu.Lock() defer s.mu.Unlock() p, ok := s.last[target] if !ok || p.Archive == "" { return "", false } return p.Archive, true } // Snapshot returns a copy of the last-proven TIMES — for the host-report gauge. func (s *RestoreTestState) Snapshot() map[string]time.Time { s.mu.Lock() defer s.mu.Unlock() out := make(map[string]time.Time, len(s.last)) for k, v := range s.last { out[k] = v.At } return out } // OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST. // // This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it // naturally prioritises a tier that has never been restore-tested at all — which on this fleet was // the offsite tier, unproven for its entire existence. // // Ties break on target id so the order is deterministic; without that, two tiers proven in the same // second would rotate by map iteration order, which is randomised in Go and would make the // behaviour untestable and occasionally starving. func (s *RestoreTestState) OldestFirst(targets []string) []string { s.mu.Lock() defer s.mu.Unlock() out := append([]string(nil), targets...) sort.SliceStable(out, func(i, j int) bool { pi, oki := s.last[out[i]] pj, okj := s.last[out[j]] ti, tj := pi.At, pj.At switch { case !oki && !okj: return out[i] < out[j] // both never proven → deterministic case !oki: return true // never proven wins case !okj: return false case !ti.Equal(tj): return ti.Before(tj) default: return out[i] < out[j] } }) return out } func (s *RestoreTestState) saveLocked() error { raw := make(map[string]provenTierJSON, len(s.last)) for target, p := range s.last { raw[target] = provenTierJSON{Archive: p.Archive, ProvenAt: p.At.UTC().Format(time.RFC3339)} } data, err := json.MarshalIndent(raw, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { return err } tmp := s.path + ".tmp" if err := os.WriteFile(tmp, data, 0o600); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, s.path) }