felhom-agent/internal/backup/backup_state.go

package backup

import (
	"encoding/json"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// BackupSuccessState persists the newest SUCCESSFUL whole-guest backup per tier and guest (R-894).
//
// Why it exists. The due-check (`localapi` handleBackupDue) asks the tier's storage when a backup last
// landed (R-84) and falls back to the in-memory record when the storage cannot be read. The in-memory
// record is empty after an agent restart (Store, R-348), so "storage unreadable" right after a restart
// read as "no record — DUE". Measured 2026-10-05 on demo-hp: the agent restarted at 04:57, the off-site
// storage answered "Can't connect" at 06:25, the 7-day tier — last copy 2026-10-01 — read DUE, the
// controller asked, and vzdump failed. This file is the last known copy the fallback reads instead.
//
// It is read ONLY when the storage cannot be read. A storage that answers is the ground truth and wins,
// in both directions: an archive found there counts, and an archive absent there is absent even when
// this file remembers a success (a pruned or deleted archive must make the tier due — the same reason
// R-84 chose the storage over a persisted record). Pinned by
// TestBackupDue_R894_SavedCopyIgnoredWhenStorageAnswers.
//
// Only SUCCESSES are written (the RestoreTestState rule): a failure must stay due and be retried, so a
// record of a failure has no reader.
type BackupSuccessState struct {
	path string
	mu   sync.Mutex
	last map[string]savedSuccess // key(target, vmid) → the newest success
}

type savedSuccess struct {
	target string
	vmid   int
	at     time.Time
	// recent (R-973) holds the start times of the newest successes (at most maxRecentSuccesses, newest
	// last) — what the local-API backup cadence cap counts, so an agent restart does not reset the cap.
	recent []time.Time
	// refusedAt (R-973) is the newest time the cap refused a backup of this guest on this tier.
	refusedAt time.Time
}

// maxRecentSuccesses bounds the per-tier success history kept for the cadence cap (R-973). The cap
// allows at most 3 runs in a window, so 8 is ample.
const maxRecentSuccesses = 8

// backupSuccessJSON is one entry on disk. Recent and LastRefusedAt are additive (R-973): a file written
// before them reads as "no history", and an older agent ignores them.
type backupSuccessJSON struct {
	Target        string   `json:"target"`
	VMID          int      `json:"vmid"`
	StartedAt     string   `json:"started_at"`
	Recent        []string `json:"recent,omitempty"`
	LastRefusedAt string   `json:"last_refused_at,omitempty"`
}

func backupStateKey(target string, vmid int) string { return target + "/" + strconv.Itoa(vmid) }

// NewBackupSuccessState opens (or creates) the state at path. A missing or unreadable file degrades to
// "nothing known" — the pre-R-894 behaviour, which is DUE — and never wedges the daemon.
func NewBackupSuccessState(path string) *BackupSuccessState {
	s := &BackupSuccessState{path: path, last: map[string]savedSuccess{}}
	data, err := os.ReadFile(path)
	if err != nil {
		return s
	}
	var entries []backupSuccessJSON
	if json.Unmarshal(data, &entries) != nil {
		return s
	}
	for _, e := range entries {
		t, perr := time.Parse(time.RFC3339, e.StartedAt)
		if perr != nil {
			continue // one unreadable entry must not lose the others
		}
		ss := savedSuccess{target: e.Target, vmid: e.VMID, at: t.UTC()}
		for _, r := range e.Recent {
			if rt, rerr := time.Parse(time.RFC3339, r); rerr == nil {
				ss.recent = append(ss.recent, rt.UTC())
			}
		}
		if len(ss.recent) == 0 {
			ss.recent = []time.Time{ss.at} // a pre-R-973 file: its one known success counts
		}
		if rt, rerr := time.Parse(time.RFC3339, e.LastRefusedAt); rerr == nil {
			ss.refusedAt = rt.UTC()
		}
		s.last[backupStateKey(e.Target, e.VMID)] = ss
	}
	return s
}

// RecordBackupSuccess saves b when it is a success newer than the one on file. target is the tier the
// job ran on (the due-check's key); a failure or an unparseable time is ignored.
func (s *BackupSuccessState) RecordBackupSuccess(target string, b hub.Backup) error {
	if s == nil || !b.Success {
		return nil
	}
	t, err := time.Parse(time.RFC3339, b.StartedAt)
	if err != nil {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	k := backupStateKey(target, b.VMID)
	old, had := s.last[k]
	if had && !t.After(old.at) {
		return nil
	}
	ss := savedSuccess{target: target, vmid: b.VMID, at: t.UTC(), refusedAt: old.refusedAt}
	ss.recent = append(append([]time.Time(nil), old.recent...), t.UTC())
	if len(ss.recent) > maxRecentSuccesses {
		ss.recent = ss.recent[len(ss.recent)-maxRecentSuccesses:]
	}
	s.last[k] = ss
	return s.saveLocked()
}

// RecentBackupSuccesses returns the saved success start times for this tier and guest that are not
// older than since (R-973 — the cadence cap's restart-proof count).
func (s *BackupSuccessState) RecentBackupSuccesses(target string, vmid int, since time.Time) []time.Time {
	if s == nil {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	var out []time.Time
	for _, t := range s.last[backupStateKey(target, vmid)].recent {
		if !t.Before(since) {
			out = append(out, t)
		}
	}
	return out
}

// RecordBackupRefusal saves that the cadence cap refused a backup of this guest on this tier at at
// (R-973), so the operator signal it raises survives an agent restart.
func (s *BackupSuccessState) RecordBackupRefusal(target string, vmid int, at time.Time) error {
	if s == nil {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	k := backupStateKey(target, vmid)
	ss, ok := s.last[k]
	if !ok {
		return nil // nothing to cap against without a success; the cap never refuses then
	}
	if !at.After(ss.refusedAt) {
		return nil
	}
	ss.refusedAt = at.UTC()
	s.last[k] = ss
	return s.saveLocked()
}

// LastBackupRefusal returns the newest saved cadence-cap refusal across every tier and guest.
func (s *BackupSuccessState) LastBackupRefusal() (target string, vmid int, at time.Time, ok bool) {
	if s == nil {
		return "", 0, time.Time{}, false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, e := range s.last {
		if !e.refusedAt.IsZero() && e.refusedAt.After(at) {
			target, vmid, at, ok = e.target, e.vmid, e.refusedAt, true
		}
	}
	return target, vmid, at, ok
}

// LastKnownSuccess returns the newest saved success for this tier and guest (ok=false = none on file).
func (s *BackupSuccessState) LastKnownSuccess(target string, vmid int) (time.Time, bool) {
	if s == nil {
		return time.Time{}, false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	e, ok := s.last[backupStateKey(target, vmid)]
	return e.at, ok
}

// KnownBackupSuccesses returns every saved success as a host-report record (the collector's
// KnownBackupReporter, so the hub keeps its evidence across a restart). Only the tier, guest and start
// time are known; the archive name is not saved and stays empty. Sorted for a stable report.
func (s *BackupSuccessState) KnownBackupSuccesses() []hub.Backup {
	if s == nil {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make([]hub.Backup, 0, len(s.last))
	for _, e := range s.last {
		out = append(out, hub.Backup{TargetID: e.target, VMID: e.vmid, Success: true, CrashConsistent: true,
			StartedAt: e.at.Format(time.RFC3339), UncoveredVolumes: []string{}})
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].TargetID != out[j].TargetID {
			return out[i].TargetID < out[j].TargetID
		}
		return out[i].VMID < out[j].VMID
	})
	return out
}

func (s *BackupSuccessState) saveLocked() error {
	entries := make([]backupSuccessJSON, 0, len(s.last))
	for _, e := range s.last {
		j := backupSuccessJSON{Target: e.target, VMID: e.vmid, StartedAt: e.at.Format(time.RFC3339)}
		for _, r := range e.recent {
			j.Recent = append(j.Recent, r.Format(time.RFC3339))
		}
		if !e.refusedAt.IsZero() {
			j.LastRefusedAt = e.refusedAt.Format(time.RFC3339)
		}
		entries = append(entries, j)
	}
	// Deterministic file content (Go's map order is random).
	sort.Slice(entries, func(i, j int) bool {
		if entries[i].Target != entries[j].Target {
			return entries[i].Target < entries[j].Target
		}
		return entries[i].VMID < entries[j].VMID
	})
	data, err := json.MarshalIndent(entries, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
		return err
	}
	tmp := s.path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, s.path)
}