felhom-agent/internal/backup/restoretest_state.go

package backup

import (
	"encoding/json"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

// RestoreTestState persists the last SUCCESSFUL restore-test per backup tier.
//
// R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth
// stating, because the two look like the same problem and are not:
//
//   - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage,
//     so the agent could ask "when did a backup last land?" and never persist anything. That is
//     strictly better, because a pruned archive correctly stops counting.
//   - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is
//     nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it
//     must be written down or it is lost.
//
// Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map
// would reset every tier to "never tested" on each restart. Ordering would then depend on map
// iteration order, and one tier could be starved indefinitely while the other is re-tested — with
// agent deploys as routine as they are, that is not a corner case.
//
// Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time
// would look freshly proven and stop being retried — the same "a failure satisfies the cadence"
// trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job:
// the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from
// ever becoming due again. The rule earns its keep twice now.
//
// R-86 (1.2) — WHICH ARCHIVE, not just when.
//
// A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is
// consistent both with "yesterday's archive is proven" and with "an archive from a week ago is
// proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION —
// a tier is due once it holds a settled archive that has not been proven — so the identity of the
// proven archive is the state, and the time is metadata (rotation ordering, operator reporting).
//
// This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT":
// here it records a result, but not WHICH result, and that is just as unable to answer the question
// being asked of it.
type RestoreTestState struct {
	path string
	mu   sync.Mutex
	last map[string]provenTier // target id → what was last PROVEN on that tier
}

// provenTier is one tier's proof: the archive that passed, and when it passed.
type provenTier struct {
	Archive string    // volid of the archive that PASSED; "" = a legacy record with no archive
	At      time.Time // when that run passed (UTC)
}

// provenTierJSON is the on-disk shape (R-86). The legacy shape was a bare RFC3339 STRING per
// target; both are read, only this one is written — see NewRestoreTestState.
type provenTierJSON struct {
	Archive  string `json:"archive"`
	ProvenAt string `json:"proven_at"`
}

// NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an
// error: it degrades to "nothing proven yet", which is the correct starting point and keeps a
// corrupt file from wedging the daemon.
//
// MIGRATION (R-86). The pre-R-86 file is `{"<target>": "<RFC3339>"}` — a time and no archive. A
// legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists
// at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after
// the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to
// read a legacy time as proof of whatever archive happens to be current, which would mark an
// unproven archive proven — inventing a guarantee out of a migration.
func NewRestoreTestState(path string) *RestoreTestState {
	s := &RestoreTestState{path: path, last: map[string]provenTier{}}
	data, err := os.ReadFile(path)
	if err != nil {
		return s
	}
	var raw map[string]json.RawMessage
	if json.Unmarshal(data, &raw) != nil {
		return s
	}
	for target, msg := range raw {
		// Legacy shape: a bare RFC3339 string.
		var legacy string
		if json.Unmarshal(msg, &legacy) == nil {
			if t, perr := time.Parse(time.RFC3339, legacy); perr == nil {
				s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately
			}
			continue
		}
		var cur provenTierJSON
		if json.Unmarshal(msg, &cur) != nil {
			continue // one unreadable entry must not lose the others
		}
		t, perr := time.Parse(time.RFC3339, cur.ProvenAt)
		if perr != nil {
			continue
		}
		s.last[target] = provenTier{Archive: cur.Archive, At: t.UTC()}
	}
	return s
}

// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed. Only call this for a
// PASSING restore-test — the archive is what makes the tier not-due, so recording one for a failed
// run would retire the archive unproven.
func (s *RestoreTestState) RecordSuccess(target, archive string, t time.Time) error {
	if target == "" {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	s.last[target] = provenTier{Archive: archive, At: t.UTC()}
	return s.saveLocked()
}

// LastSuccess returns when this tier was last proven (ok=false = never).
func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	p, ok := s.last[target]
	return p.At, ok
}

// ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested,
// or a legacy record carrying only a time). It is the due-check's whole question: an archive that is
// not this one has not been proven.
func (s *RestoreTestState) ProvenArchive(target string) (string, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	p, ok := s.last[target]
	if !ok || p.Archive == "" {
		return "", false
	}
	return p.Archive, true
}

// Snapshot returns a copy of the last-proven TIMES — for the host-report gauge.
func (s *RestoreTestState) Snapshot() map[string]time.Time {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make(map[string]time.Time, len(s.last))
	for k, v := range s.last {
		out[k] = v.At
	}
	return out
}

// OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST.
//
// This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it
// naturally prioritises a tier that has never been restore-tested at all — which on this fleet was
// the offsite tier, unproven for its entire existence.
//
// Ties break on target id so the order is deterministic; without that, two tiers proven in the same
// second would rotate by map iteration order, which is randomised in Go and would make the
// behaviour untestable and occasionally starving.
func (s *RestoreTestState) OldestFirst(targets []string) []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := append([]string(nil), targets...)
	sort.SliceStable(out, func(i, j int) bool {
		pi, oki := s.last[out[i]]
		pj, okj := s.last[out[j]]
		ti, tj := pi.At, pj.At
		switch {
		case !oki && !okj:
			return out[i] < out[j] // both never proven → deterministic
		case !oki:
			return true // never proven wins
		case !okj:
			return false
		case !ti.Equal(tj):
			return ti.Before(tj)
		default:
			return out[i] < out[j]
		}
	})
	return out
}

func (s *RestoreTestState) saveLocked() error {
	raw := make(map[string]provenTierJSON, len(s.last))
	for target, p := range s.last {
		raw[target] = provenTierJSON{Archive: p.Archive, ProvenAt: p.At.UTC().Format(time.RFC3339)}
	}
	data, err := json.MarshalIndent(raw, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
		return err
	}
	tmp := s.path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, s.path)
}