felhom-agent/internal/backup/restoretest_state.go

package backup

import (
	"encoding/json"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

// RestoreTestState persists the last SUCCESSFUL restore-test per backup tier.
//
// R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth
// stating, because the two look like the same problem and are not:
//
//   - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage,
//     so the agent could ask "when did a backup last land?" and never persist anything. That is
//     strictly better, because a pruned archive correctly stops counting.
//   - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is
//     nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it
//     must be written down or it is lost.
//
// Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map
// would reset every tier to "never tested" on each restart. Ordering would then depend on map
// iteration order, and one tier could be starved indefinitely while the other is re-tested — with
// agent deploys as routine as they are, that is not a corner case.
//
// Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time
// would look freshly proven and stop being retried — the same "a failure satisfies the cadence"
// trap the backup due-check avoids.
type RestoreTestState struct {
	path string
	mu   sync.Mutex
	last map[string]time.Time // target id → last SUCCESSFUL restore-test (UTC)
}

// NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an
// error: it degrades to "nothing proven yet", which is the correct starting point and keeps a
// corrupt file from wedging the daemon.
func NewRestoreTestState(path string) *RestoreTestState {
	s := &RestoreTestState{path: path, last: map[string]time.Time{}}
	data, err := os.ReadFile(path)
	if err != nil {
		return s
	}
	var raw map[string]string
	if json.Unmarshal(data, &raw) != nil {
		return s
	}
	for target, ts := range raw {
		if t, perr := time.Parse(time.RFC3339, ts); perr == nil {
			s.last[target] = t.UTC()
		}
	}
	return s
}

// RecordSuccess stamps a tier as proven at t. Only call this for a PASSING restore-test.
func (s *RestoreTestState) RecordSuccess(target string, t time.Time) error {
	if target == "" {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	s.last[target] = t.UTC()
	return s.saveLocked()
}

// LastSuccess returns when this tier was last proven (ok=false = never).
func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	t, ok := s.last[target]
	return t, ok
}

// Snapshot returns a copy of the whole map — for the host-report gauge.
func (s *RestoreTestState) Snapshot() map[string]time.Time {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make(map[string]time.Time, len(s.last))
	for k, v := range s.last {
		out[k] = v
	}
	return out
}

// OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST.
//
// This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it
// naturally prioritises a tier that has never been restore-tested at all — which on this fleet was
// the offsite tier, unproven for its entire existence.
//
// Ties break on target id so the order is deterministic; without that, two tiers proven in the same
// second would rotate by map iteration order, which is randomised in Go and would make the
// behaviour untestable and occasionally starving.
func (s *RestoreTestState) OldestFirst(targets []string) []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := append([]string(nil), targets...)
	sort.SliceStable(out, func(i, j int) bool {
		ti, oki := s.last[out[i]]
		tj, okj := s.last[out[j]]
		switch {
		case !oki && !okj:
			return out[i] < out[j] // both never proven → deterministic
		case !oki:
			return true // never proven wins
		case !okj:
			return false
		case !ti.Equal(tj):
			return ti.Before(tj)
		default:
			return out[i] < out[j]
		}
	})
	return out
}

func (s *RestoreTestState) saveLocked() error {
	raw := make(map[string]string, len(s.last))
	for target, t := range s.last {
		raw[target] = t.UTC().Format(time.RFC3339)
	}
	data, err := json.MarshalIndent(raw, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
		return err
	}
	tmp := s.path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, s.path)
}