felhom-agent/internal/backup/restoretest_due.go

package backup

import (
	"context"
	"fmt"
	"time"
)

// R-86 — a restore-test follows the BACKUP, not the clock.
//
// ── WHAT WAS WRONG ───────────────────────────────────────────────────────────────────────────
//
// The trigger was `time.NewTicker(cadence)` started at daemon start, and the tier was chosen by
// oldest-proven rotation. Its phase was therefore the PROCESS'S UPTIME: agent deploys are routine,
// so the test drifted to an arbitrary time of day every week; a fresh archive could sit unproven
// while an older one was re-tested; and a weekly tier was tested on the same rhythm as a daily one,
// sometimes twice on the same archive.
//
// ── THE RULE, AND THE TRAP IN ITS OBVIOUS FORM ───────────────────────────────────────────────
//
// R-86's ask reads "test a tier ~24 h after its own newest archive". Implemented literally —
// *"due when the newest archive is at least `settle` old"* — a DAILY tier is NEVER due: a new
// archive lands every day, so the newest archive's age resets to zero long before it reaches 24 h.
// The naive rule silently switches restore-testing off for the tier that matters most, and it is
// the version a reasonable person would write. It has a red-proof of its own
// (TestDue_NaiveNewestArchiveAgeRuleNeverFiresOnADailyTier).
//
// The rule implemented here:
//
//	Let A = the newest archive on this tier that is at least `settle` old.
//	The tier is DUE when A exists and A HAS NOT ALREADY BEEN PROVEN.
//
//	daily tier    → A is yesterday's archive; a new one settles each day  → proved once per day
//	weekly tier   → A is last week's until the next settles               → proved once per week
//	newborn tier  → A does not exist                                      → UNKNOWN, never a fault
//
// Per-archive due-ness IS the pacing: one test per archive generation and no more. There is
// deliberately no second rate limiter on top of it (§8.4) — two independent pacing mechanisms
// produce a cadence nobody can predict from either.
//
// ── WHAT DID NOT CHANGE ──────────────────────────────────────────────────────────────────────
//
// The one-heavy-operation gate, the success-only proof credit, the oldest-proven ordering (now the
// tie-break between two DUE tiers), the restore-test itself, its journal and its scratch band. Only
// the trigger changed.

// DueVerdict is one tier's due-ness, and the evidence for it. Every field is logged: a due-check
// that cannot say WHY is a quiet path, and quiet paths are what this monitor family keeps shipping.
type DueVerdict struct {
	Target string // the tier's storage target id

	// Due is true only when Archive is set and has not been proven.
	Due bool
	// Archive is the settled candidate A ("" when the tier holds none).
	Archive string
	// Landed is when A landed on the tier (zero when Archive is "").
	Landed time.Time
	// ProvenArchive is what the state says was last proven on this tier ("" = nothing/legacy).
	ProvenArchive string
	// Err is a candidate-lookup failure. A tier whose archives cannot be listed is UNKNOWN — it is
	// NEVER reported as "not due", which would silently retire a tier the moment its storage
	// stopped answering. Due stays false (we have no archive to test) and the error travels.
	Err error
	// Reason is the one-line human account of this verdict.
	Reason string
}

// String renders a verdict for the operator log / selftest output.
func (v DueVerdict) String() string {
	return fmt.Sprintf("tier=%s due=%v archive=%q reason=%s", v.Target, v.Due, v.Archive, v.Reason)
}

// EvaluateDue returns the due verdict for every configured tier, ordered oldest-proven first.
//
// Ordering is the R-85 rotation, demoted to a TIE-BREAK: it no longer decides whether a test
// happens (due-ness does), only which of several due tiers goes first. Keeping it means a tier can
// still never be starved — a tier that has waited longest is served first — and keeping it as the
// order rather than as the trigger is the whole of this change.
func (s *Scheduler) EvaluateDue(ctx context.Context) []DueVerdict {
	if !s.rotating() {
		return nil
	}
	order := s.tiers
	if s.rtState != nil {
		order = s.rtState.OldestFirst(s.tiers)
	}
	cutoff := s.settleCutoff()
	out := make([]DueVerdict, 0, len(order))
	for _, target := range order {
		out = append(out, s.evaluateTier(ctx, target, cutoff))
	}
	return out
}

// settleCutoff is the newest landing time an archive may have and still count as settled.
func (s *Scheduler) settleCutoff() time.Time {
	if s.settle <= 0 {
		return time.Time{} // no settle requirement configured → any archive is a candidate
	}
	return s.now().Add(-s.settle)
}

// evaluateTier is the per-tier due-check. PURE given the picker and the state, so the rule is
// unit-tested directly rather than inferred from whether a fake runner happened to be called.
func (s *Scheduler) evaluateTier(ctx context.Context, target string, cutoff time.Time) DueVerdict {
	v := DueVerdict{Target: target}
	archive, landed, err := s.tierPick(ctx, target, cutoff)
	if err != nil {
		// UNKNOWN, never "not due", and never silent.
		v.Err = err
		v.Reason = fmt.Sprintf("candidate lookup FAILED (%v) — tier is unknown this evaluation, not proven and not dismissed", err)
		return v
	}
	v.Archive, v.Landed = archive, landed
	if archive == "" {
		v.Reason = "no settled archive yet — nothing to prove (newborn or still settling)"
		return v
	}
	proven, ok := "", false
	if s.rtState != nil {
		proven, ok = s.rtState.ProvenArchive(target)
	}
	v.ProvenArchive = proven
	if ok && proven == archive {
		v.Reason = fmt.Sprintf("newest settled archive (landed %s) is already proven", landed.Format(time.RFC3339))
		return v
	}
	v.Due = true
	switch {
	case !ok && proven == "":
		v.Reason = fmt.Sprintf("newest settled archive (landed %s) has not been proven; nothing proven on this tier yet", landed.Format(time.RFC3339))
	default:
		v.Reason = fmt.Sprintf("newest settled archive (landed %s) has not been proven (last proven archive was a different one)", landed.Format(time.RFC3339))
	}
	return v
}

// EvaluateDueTier is EvaluateDue for ONE named tier — the selftest's per-tier cost probe, so the
// WAN leg of an offsite lookup is attributable rather than buried in an aggregate.
func (s *Scheduler) EvaluateDueTier(ctx context.Context, target string) DueVerdict {
	return s.evaluateTier(ctx, target, s.settleCutoff())
}

// verdictSummary renders one compact line of per-tier verdicts for the "nothing due" log.
//
// It re-evaluates rather than threading the verdicts out of pickForThisRun, and that is a
// deliberate trade: this runs only on the path where NOTHING is due, so the cost is one extra
// storage listing per tier on an otherwise idle evaluation (measured 18 ms local / 392 ms offsite,
// R-86 Part 1.4), and in exchange the logging path cannot drift from the deciding path by holding a
// stale copy of it. If that cost ever matters, pass the verdicts in — do not let the two diverge.
func (s *Scheduler) verdictSummary(ctx context.Context) string {
	out := ""
	for _, v := range s.EvaluateDue(ctx) {
		if out != "" {
			out += "; "
		}
		switch {
		case v.Err != nil:
			out += v.Target + ": UNKNOWN (" + v.Err.Error() + ")"
		default:
			out += v.Target + ": " + v.Reason
		}
	}
	if out == "" {
		return "no tiers configured"
	}
	return out
}