package backup import ( "context" "encoding/json" "os" "path/filepath" "sort" "sync" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // RestoreTestState persists the last SUCCESSFUL restore-test per backup tier. // // R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth // stating, because the two look like the same problem and are not: // // - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage, // so the agent could ask "when did a backup last land?" and never persist anything. That is // strictly better, because a pruned archive correctly stops counting. // - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is // nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it // must be written down or it is lost. // // Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map // would reset every tier to "never tested" on each restart. Ordering would then depend on map // iteration order, and one tier could be starved indefinitely while the other is re-tested — with // agent deploys as routine as they are, that is not a corner case. // // Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time // would look freshly proven and stop being retried — the same "a failure satisfies the cadence" // trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job: // the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from // ever becoming due again. The rule earns its keep twice now. // // R-86 (1.2) — WHICH ARCHIVE, not just when. // // A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is // consistent both with "yesterday's archive is proven" and with "an archive from a week ago is // proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION — // a tier is due once it holds a settled archive that has not been proven — so the identity of the // proven archive is the state, and the time is metadata (rotation ordering, operator reporting). // // This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT": // here it records a result, but not WHICH result, and that is just as unable to answer the question // being asked of it. type RestoreTestState struct { path string mu sync.Mutex last map[string]provenTier // target id → what was last PROVEN on that tier } // provenTier is one tier's proof: the archive that passed, which tier it was, what was verified, // and when. // // R-189 added `Tier` and `Verified`. Until then this record could answer the DUE-check but could not // be REPORTED, and being reportable is what closes R-189: a proof held only in the in-memory result // store vanishes on restart, and under per-archive due-ness the box will not repeat the work, so the // hub can stay ignorant of a real success until the next archive generation. // // `Tier` is stored rather than derived because it is known for certain at proof time (the run's own // spec used it to choose the restore timeout) and deriving it later would need a storage-type lookup // at report-building time — a network call that can fail, on a path where failing means mis-labelling // a proof. Store what you knew when you knew it. type provenTier struct { Archive string // volid of the archive that PASSED; "" = a legacy record with no archive Tier string // "local" | "pbs" — as the run reported it; "" = pre-R-189 record Verified string // what the run verified (e.g. "boot+running"); "" = pre-R-189 record At time.Time // when that run passed (UTC) } // reportable reports whether this record can be re-reported to the hub as a restore-test result. // // It needs BOTH the archive and the tier: the hub keys its edge-triggered failure state on the // archive and its per-tier proof lookup on the tier, so an entry missing either is not a usable // proof — and emitting one anyway would be a report the hub cannot act on, dressed as evidence. // A pre-R-189 record is therefore silently not reported; the tier's next real proof fills it in. func (p provenTier) reportable() bool { return p.Archive != "" && p.Tier != "" } // provenTierJSON is the on-disk shape. Two older shapes are read and neither is written: // // v1 (pre-R-86) "": "" — a time, no archive // v2 (R-86) "": {archive, proven_at} — due-check usable, not reportable // v3 (R-189) "": {archive, tier, verified, …} — both // // Fields absent in an older file unmarshal to "", which is exactly the "no usable proof" signal the // readers above test for — the migration needs no version number because the absence IS the answer. type provenTierJSON struct { Archive string `json:"archive"` Tier string `json:"tier,omitempty"` Verified string `json:"verified,omitempty"` ProvenAt string `json:"proven_at"` } // NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an // error: it degrades to "nothing proven yet", which is the correct starting point and keeps a // corrupt file from wedging the daemon. // // MIGRATION (R-86). The pre-R-86 file is `{"": ""}` — a time and no archive. A // legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists // at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after // the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to // read a legacy time as proof of whatever archive happens to be current, which would mark an // unproven archive proven — inventing a guarantee out of a migration. func NewRestoreTestState(path string) *RestoreTestState { s := &RestoreTestState{path: path, last: map[string]provenTier{}} data, err := os.ReadFile(path) if err != nil { return s } var raw map[string]json.RawMessage if json.Unmarshal(data, &raw) != nil { return s } for target, msg := range raw { // Legacy shape: a bare RFC3339 string. var legacy string if json.Unmarshal(msg, &legacy) == nil { if t, perr := time.Parse(time.RFC3339, legacy); perr == nil { s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately } continue } var cur provenTierJSON if json.Unmarshal(msg, &cur) != nil { continue // one unreadable entry must not lose the others } t, perr := time.Parse(time.RFC3339, cur.ProvenAt) if perr != nil { continue } s.last[target] = provenTier{Archive: cur.Archive, Tier: cur.Tier, Verified: cur.Verified, At: t.UTC()} } return s } // RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed, the TIER the run // reported, and what it verified. Only call this for a PASSING restore-test — the archive is what // makes the tier not-due, so recording one for a failed run would retire the archive unproven. // // ONLY SUCCESSES ARE PERSISTED, AND THE ASYMMETRY IS DELIBERATE (R-189 §8.1). Say it here because // the next reader will notice failures are absent and try to "fix" it: // // a SUCCESS suppresses future work — a proven archive is never re-tested, so a lost proof leaves // the system quietly less tested than it believes. It must survive a restart. // // a FAILURE causes future work — a failing tier stays due and is retried at the next evaluation, // so a lost failure heals itself within one interval. Persisting it would do the opposite of // helping: a healed tier would keep reporting a failure that is no longer true. func (s *RestoreTestState) RecordSuccess(target, archive, tier, verified string, t time.Time) error { if target == "" { return nil } s.mu.Lock() defer s.mu.Unlock() s.last[target] = provenTier{Archive: archive, Tier: tier, Verified: verified, At: t.UTC()} return s.saveLocked() } // LastSuccess returns when this tier was last proven (ok=false = never). func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) { s.mu.Lock() defer s.mu.Unlock() p, ok := s.last[target] return p.At, ok } // ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested, // or a legacy record carrying only a time). It is the due-check's whole question: an archive that is // not this one has not been proven. func (s *RestoreTestState) ProvenArchive(target string) (string, bool) { s.mu.Lock() defer s.mu.Unlock() p, ok := s.last[target] if !ok || p.Archive == "" { return "", false } return p.Archive, true } // Snapshot returns a copy of the last-proven TIMES. // // It carried the comment "for the host-report gauge" from the day it was written and **had no caller // at all** until R-189 — a seam built and never wired, and an invariant asserted in a comment with // nothing pinning it, in one method. The host report is now fed by ProvenRestoreTests below, which // carries the archive and the tier that a bare timestamp cannot. This stays for callers that want // only the times; if it acquires none, delete it rather than let it claim a purpose again. func (s *RestoreTestState) Snapshot() map[string]time.Time { s.mu.Lock() defer s.mu.Unlock() out := make(map[string]time.Time, len(s.last)) for k, v := range s.last { out[k] = v.At } return out } // ProvenRestoreTests renders the persisted proofs as host-report entries — the R-189 fix. // // It satisfies hub.RestoreTestReporter's shape, so the collector can merge these with the in-memory // results. What it emits is a RE-REPORT of a run that really happened, not a synthesis: // // - `Pass` is true because ONLY successes are stored (RecordSuccess is the sole writer); // - `SourceArchive`, `SourceTier`, `Verified` and `TestedAt` are the values that run reported; // - the run mechanics (scratch VMID, duration, warnings) are NOT re-invented. An absent duration // is not a claim; a fabricated one would be. // // A record that cannot be reported honestly is omitted rather than padded — see provenTier.reportable. // **A tier with no usable proof produces NO entry**: an unproven tier reading as proven would be a // worse defect than the one this fixes. func (s *RestoreTestState) ProvenRestoreTests(context.Context) []hub.RestoreTest { s.mu.Lock() defer s.mu.Unlock() out := make([]hub.RestoreTest, 0, len(s.last)) for _, p := range s.last { if !p.reportable() { continue } out = append(out, hub.RestoreTest{ SourceArchive: p.Archive, SourceTier: p.Tier, Pass: true, Verified: p.Verified, TestedAt: p.At.UTC().Format(time.RFC3339), }) } // Deterministic order: the report is compared byte-wise by the contract test, and Go's map // iteration is randomised. sort.Slice(out, func(i, j int) bool { return out[i].SourceTier < out[j].SourceTier }) return out } // OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST. // // This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it // naturally prioritises a tier that has never been restore-tested at all — which on this fleet was // the offsite tier, unproven for its entire existence. // // Ties break on target id so the order is deterministic; without that, two tiers proven in the same // second would rotate by map iteration order, which is randomised in Go and would make the // behaviour untestable and occasionally starving. func (s *RestoreTestState) OldestFirst(targets []string) []string { s.mu.Lock() defer s.mu.Unlock() out := append([]string(nil), targets...) sort.SliceStable(out, func(i, j int) bool { pi, oki := s.last[out[i]] pj, okj := s.last[out[j]] ti, tj := pi.At, pj.At switch { case !oki && !okj: return out[i] < out[j] // both never proven → deterministic case !oki: return true // never proven wins case !okj: return false case !ti.Equal(tj): return ti.Before(tj) default: return out[i] < out[j] } }) return out } func (s *RestoreTestState) saveLocked() error { raw := make(map[string]provenTierJSON, len(s.last)) for target, p := range s.last { raw[target] = provenTierJSON{ Archive: p.Archive, Tier: p.Tier, Verified: p.Verified, ProvenAt: p.At.UTC().Format(time.RFC3339), } } data, err := json.MarshalIndent(raw, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { return err } tmp := s.path + ".tmp" if err := os.WriteFile(tmp, data, 0o600); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, s.path) }