package backup import ( "encoding/json" "os" "path/filepath" "sort" "sync" "time" ) // RestoreTestState persists the last SUCCESSFUL restore-test per backup tier. // // R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth // stating, because the two look like the same problem and are not: // // - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage, // so the agent could ask "when did a backup last land?" and never persist anything. That is // strictly better, because a pruned archive correctly stops counting. // - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is // nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it // must be written down or it is lost. // // Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map // would reset every tier to "never tested" on each restart. Ordering would then depend on map // iteration order, and one tier could be starved indefinitely while the other is re-tested — with // agent deploys as routine as they are, that is not a corner case. // // Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time // would look freshly proven and stop being retried — the same "a failure satisfies the cadence" // trap the backup due-check avoids. type RestoreTestState struct { path string mu sync.Mutex last map[string]time.Time // target id → last SUCCESSFUL restore-test (UTC) } // NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an // error: it degrades to "nothing proven yet", which is the correct starting point and keeps a // corrupt file from wedging the daemon. func NewRestoreTestState(path string) *RestoreTestState { s := &RestoreTestState{path: path, last: map[string]time.Time{}} data, err := os.ReadFile(path) if err != nil { return s } var raw map[string]string if json.Unmarshal(data, &raw) != nil { return s } for target, ts := range raw { if t, perr := time.Parse(time.RFC3339, ts); perr == nil { s.last[target] = t.UTC() } } return s } // RecordSuccess stamps a tier as proven at t. Only call this for a PASSING restore-test. func (s *RestoreTestState) RecordSuccess(target string, t time.Time) error { if target == "" { return nil } s.mu.Lock() defer s.mu.Unlock() s.last[target] = t.UTC() return s.saveLocked() } // LastSuccess returns when this tier was last proven (ok=false = never). func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) { s.mu.Lock() defer s.mu.Unlock() t, ok := s.last[target] return t, ok } // Snapshot returns a copy of the whole map — for the host-report gauge. func (s *RestoreTestState) Snapshot() map[string]time.Time { s.mu.Lock() defer s.mu.Unlock() out := make(map[string]time.Time, len(s.last)) for k, v := range s.last { out[k] = v } return out } // OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST. // // This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it // naturally prioritises a tier that has never been restore-tested at all — which on this fleet was // the offsite tier, unproven for its entire existence. // // Ties break on target id so the order is deterministic; without that, two tiers proven in the same // second would rotate by map iteration order, which is randomised in Go and would make the // behaviour untestable and occasionally starving. func (s *RestoreTestState) OldestFirst(targets []string) []string { s.mu.Lock() defer s.mu.Unlock() out := append([]string(nil), targets...) sort.SliceStable(out, func(i, j int) bool { ti, oki := s.last[out[i]] tj, okj := s.last[out[j]] switch { case !oki && !okj: return out[i] < out[j] // both never proven → deterministic case !oki: return true // never proven wins case !okj: return false case !ti.Equal(tj): return ti.Before(tj) default: return out[i] < out[j] } }) return out } func (s *RestoreTestState) saveLocked() error { raw := make(map[string]string, len(s.last)) for target, t := range s.last { raw[target] = t.UTC().Format(time.RFC3339) } data, err := json.MarshalIndent(raw, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { return err } tmp := s.path + ".tmp" if err := os.WriteFile(tmp, data, 0o600); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, s.path) }