felhom-agent/internal/backup/restoretest_state.go

package backup

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// RestoreTestState persists the last SUCCESSFUL restore-test per backup tier.
//
// R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth
// stating, because the two look like the same problem and are not:
//
//   - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage,
//     so the agent could ask "when did a backup last land?" and never persist anything. That is
//     strictly better, because a pruned archive correctly stops counting.
//   - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is
//     nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it
//     must be written down or it is lost.
//
// Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map
// would reset every tier to "never tested" on each restart. Ordering would then depend on map
// iteration order, and one tier could be starved indefinitely while the other is re-tested — with
// agent deploys as routine as they are, that is not a corner case.
//
// Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time
// would look freshly proven and stop being retried — the same "a failure satisfies the cadence"
// trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job:
// the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from
// ever becoming due again. The rule earns its keep twice now.
//
// R-86 (1.2) — WHICH ARCHIVE, not just when.
//
// A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is
// consistent both with "yesterday's archive is proven" and with "an archive from a week ago is
// proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION —
// a tier is due once it holds a settled archive that has not been proven — so the identity of the
// proven archive is the state, and the time is metadata (rotation ordering, operator reporting).
//
// This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT":
// here it records a result, but not WHICH result, and that is just as unable to answer the question
// being asked of it.
type RestoreTestState struct {
	path string
	mu   sync.Mutex
	last map[string]provenTier // target id → what was last PROVEN on that tier
}

// provenTier is one tier's proof: the archive that passed, which tier it was, what was verified,
// and when.
//
// R-189 added `Tier` and `Verified`. Until then this record could answer the DUE-check but could not
// be REPORTED, and being reportable is what closes R-189: a proof held only in the in-memory result
// store vanishes on restart, and under per-archive due-ness the box will not repeat the work, so the
// hub can stay ignorant of a real success until the next archive generation.
//
// `Tier` is stored rather than derived because it is known for certain at proof time (the run's own
// spec used it to choose the restore timeout) and deriving it later would need a storage-type lookup
// at report-building time — a network call that can fail, on a path where failing means mis-labelling
// a proof. Store what you knew when you knew it.
type provenTier struct {
	Archive  string    // volid of the archive that PASSED; "" = a legacy record with no archive
	Tier     string    // "local" | "pbs" — as the run reported it; "" = pre-R-189 record
	Verified string    // what the run verified (e.g. "boot+running"); "" = pre-R-189 record
	At       time.Time // when that run passed (UTC)
}

// reportable reports whether this record can be re-reported to the hub as a restore-test result.
//
// It needs BOTH the archive and the tier: the hub keys its edge-triggered failure state on the
// archive and its per-tier proof lookup on the tier, so an entry missing either is not a usable
// proof — and emitting one anyway would be a report the hub cannot act on, dressed as evidence.
// A pre-R-189 record is therefore silently not reported; the tier's next real proof fills it in.
func (p provenTier) reportable() bool { return p.Archive != "" && p.Tier != "" }

// provenTierJSON is the on-disk shape. Two older shapes are read and neither is written:
//
//	v1 (pre-R-86)  "<target>": "<RFC3339>"                     — a time, no archive
//	v2 (R-86)      "<target>": {archive, proven_at}            — due-check usable, not reportable
//	v3 (R-189)     "<target>": {archive, tier, verified, …}    — both
//
// Fields absent in an older file unmarshal to "", which is exactly the "no usable proof" signal the
// readers above test for — the migration needs no version number because the absence IS the answer.
type provenTierJSON struct {
	Archive  string `json:"archive"`
	Tier     string `json:"tier,omitempty"`
	Verified string `json:"verified,omitempty"`
	ProvenAt string `json:"proven_at"`
}

// NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an
// error: it degrades to "nothing proven yet", which is the correct starting point and keeps a
// corrupt file from wedging the daemon.
//
// MIGRATION (R-86). The pre-R-86 file is `{"<target>": "<RFC3339>"}` — a time and no archive. A
// legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists
// at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after
// the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to
// read a legacy time as proof of whatever archive happens to be current, which would mark an
// unproven archive proven — inventing a guarantee out of a migration.
func NewRestoreTestState(path string) *RestoreTestState {
	s := &RestoreTestState{path: path, last: map[string]provenTier{}}
	data, err := os.ReadFile(path)
	if err != nil {
		return s
	}
	var raw map[string]json.RawMessage
	if json.Unmarshal(data, &raw) != nil {
		return s
	}
	for target, msg := range raw {
		// Legacy shape: a bare RFC3339 string.
		var legacy string
		if json.Unmarshal(msg, &legacy) == nil {
			if t, perr := time.Parse(time.RFC3339, legacy); perr == nil {
				s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately
			}
			continue
		}
		var cur provenTierJSON
		if json.Unmarshal(msg, &cur) != nil {
			continue // one unreadable entry must not lose the others
		}
		t, perr := time.Parse(time.RFC3339, cur.ProvenAt)
		if perr != nil {
			continue
		}
		s.last[target] = provenTier{Archive: cur.Archive, Tier: cur.Tier, Verified: cur.Verified, At: t.UTC()}
	}
	return s
}

// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed, the TIER the run
// reported, and what it verified. Only call this for a PASSING restore-test — the archive is what
// makes the tier not-due, so recording one for a failed run would retire the archive unproven.
//
// ONLY SUCCESSES ARE PERSISTED, AND THE ASYMMETRY IS DELIBERATE (R-189 §8.1). Say it here because
// the next reader will notice failures are absent and try to "fix" it:
//
//	a SUCCESS suppresses future work — a proven archive is never re-tested, so a lost proof leaves
//	the system quietly less tested than it believes. It must survive a restart.
//
//	a FAILURE causes future work — a failing tier stays due and is retried at the next evaluation,
//	so a lost failure heals itself within one interval. Persisting it would do the opposite of
//	helping: a healed tier would keep reporting a failure that is no longer true.
func (s *RestoreTestState) RecordSuccess(target, archive, tier, verified string, t time.Time) error {
	if target == "" {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	s.last[target] = provenTier{Archive: archive, Tier: tier, Verified: verified, At: t.UTC()}
	return s.saveLocked()
}

// LastSuccess returns when this tier was last proven (ok=false = never).
func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	p, ok := s.last[target]
	return p.At, ok
}

// ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested,
// or a legacy record carrying only a time). It is the due-check's whole question: an archive that is
// not this one has not been proven.
func (s *RestoreTestState) ProvenArchive(target string) (string, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	p, ok := s.last[target]
	if !ok || p.Archive == "" {
		return "", false
	}
	return p.Archive, true
}

// Snapshot returns a copy of the last-proven TIMES.
//
// It carried the comment "for the host-report gauge" from the day it was written and **had no caller
// at all** until R-189 — a seam built and never wired, and an invariant asserted in a comment with
// nothing pinning it, in one method. The host report is now fed by ProvenRestoreTests below, which
// carries the archive and the tier that a bare timestamp cannot. This stays for callers that want
// only the times; if it acquires none, delete it rather than let it claim a purpose again.
func (s *RestoreTestState) Snapshot() map[string]time.Time {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make(map[string]time.Time, len(s.last))
	for k, v := range s.last {
		out[k] = v.At
	}
	return out
}

// ProvenRestoreTests renders the persisted proofs as host-report entries — the R-189 fix.
//
// It satisfies hub.RestoreTestReporter's shape, so the collector can merge these with the in-memory
// results. What it emits is a RE-REPORT of a run that really happened, not a synthesis:
//
//   - `Pass` is true because ONLY successes are stored (RecordSuccess is the sole writer);
//   - `SourceArchive`, `SourceTier`, `Verified` and `TestedAt` are the values that run reported;
//   - the run mechanics (scratch VMID, duration, warnings) are NOT re-invented. An absent duration
//     is not a claim; a fabricated one would be.
//
// A record that cannot be reported honestly is omitted rather than padded — see provenTier.reportable.
// **A tier with no usable proof produces NO entry**: an unproven tier reading as proven would be a
// worse defect than the one this fixes.
func (s *RestoreTestState) ProvenRestoreTests(context.Context) []hub.RestoreTest {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make([]hub.RestoreTest, 0, len(s.last))
	for _, p := range s.last {
		if !p.reportable() {
			continue
		}
		out = append(out, hub.RestoreTest{
			SourceArchive: p.Archive,
			SourceTier:    p.Tier,
			Pass:          true,
			Verified:      p.Verified,
			TestedAt:      p.At.UTC().Format(time.RFC3339),
		})
	}
	// Deterministic order: the report is compared byte-wise by the contract test, and Go's map
	// iteration is randomised.
	sort.Slice(out, func(i, j int) bool { return out[i].SourceTier < out[j].SourceTier })
	return out
}

// OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST.
//
// This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it
// naturally prioritises a tier that has never been restore-tested at all — which on this fleet was
// the offsite tier, unproven for its entire existence.
//
// Ties break on target id so the order is deterministic; without that, two tiers proven in the same
// second would rotate by map iteration order, which is randomised in Go and would make the
// behaviour untestable and occasionally starving.
func (s *RestoreTestState) OldestFirst(targets []string) []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := append([]string(nil), targets...)
	sort.SliceStable(out, func(i, j int) bool {
		pi, oki := s.last[out[i]]
		pj, okj := s.last[out[j]]
		ti, tj := pi.At, pj.At
		switch {
		case !oki && !okj:
			return out[i] < out[j] // both never proven → deterministic
		case !oki:
			return true // never proven wins
		case !okj:
			return false
		case !ti.Equal(tj):
			return ti.Before(tj)
		default:
			return out[i] < out[j]
		}
	})
	return out
}

func (s *RestoreTestState) saveLocked() error {
	raw := make(map[string]provenTierJSON, len(s.last))
	for target, p := range s.last {
		raw[target] = provenTierJSON{
			Archive: p.Archive, Tier: p.Tier, Verified: p.Verified,
			ProvenAt: p.At.UTC().Format(time.RFC3339),
		}
	}
	data, err := json.MarshalIndent(raw, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
		return err
	}
	tmp := s.path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, s.path)
}