felhom-agent/internal/backup/schedule.go

package backup

import (
	"context"
	"log/slog"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
)

// RestoreTestRunner is the reconcile-engine seam the scheduler drives (*reconcile.Engine
// satisfies it). Kept narrow so the scheduler is unit-testable with a fake.
type RestoreTestRunner interface {
	RunRestoreTest(ctx context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult
}

// CandidatePicker resolves the archive volid to restore-test (newest backup), or "" when
// there is none yet (the tick then no-ops).
type CandidatePicker func(ctx context.Context) (string, error)

// SpecBuilder yields the RestoreTestSpec for ONE run, given the archive that was picked.
//
// R-85 (1.1): this REPLACES a frozen spec value. It used to be built by an immediately-invoked
// function at daemon start, so `storageTier()` and `restoreTaskTimeout()` were evaluated ONCE and
// the resulting value reused for every run for the lifetime of the process. Two consequences:
//   - nothing tier-varying was expressible at all (the offsite tier could never be scheduled), and
//   - it was a latent staleness bug in its own right — a storage-type or config change did not take
//     effect until the daemon restarted.
//
// The archive is passed in because the tier MUST be derived from it (the v0.100.0 rule), never from
// the configured target: deriving it from config is what produced the 600 s false failure when a
// PBS archive was classified "local" and got the 10-minute local wait.
type SpecBuilder func(ctx context.Context, archive string) reconcile.RestoreTestSpec

// TierPicker resolves the newest archive on a NAMED tier that landed AT OR BEFORE notAfter (the
// settle cutoff), together with when it landed. (*BackupRunner).PickSettledRestoreCandidateOn
// satisfies it. A zero notAfter means "no settle requirement".
//
// R-86 widened this seam from (target) → archive. The landing time is what makes the due-check's
// verdict explainable — "archive X, which landed at T, has not been proven" — and the cutoff is
// what makes the rule per-ARCHIVE-GENERATION instead of per-interval. "" must NOT be an error: a
// brand-new offsite tier legitimately has nothing to restore yet.
type TierPicker func(ctx context.Context, target string, notAfter time.Time) (archive string, landed time.Time, err error)

// Scheduler runs the self-restore-test on an agent-internal cadence. It is the fourth daemon
// goroutine; it does real restore→boot→destroy, so it only runs when the cadence is enabled
// AND a valid scratch band is configured (validated by the caller before construction).
type Scheduler struct {
	runner RestoreTestRunner
	pick   CandidatePicker
	store  *Store
	spec   SpecBuilder // R-85: evaluated PER RUN, never frozen at construction
	// cadence is the EVALUATION interval (R-86) — how often "is anything due?" is asked. It is no
	// longer the thing that decides a test happens; see restoretest_due.go.
	cadence time.Duration
	// settle is how long an archive must have sat before it is a candidate (R-86).
	settle time.Duration
	logger *slog.Logger
	now    func() time.Time

	// R-85 tier rotation. All optional: without them the scheduler behaves exactly as before
	// (single tier via `pick`), which keeps every existing caller and test working untouched.
	tiers    []string          // configured tier target ids, primary first
	tierPick TierPicker        // newest archive on a named tier
	rtState  *RestoreTestState // persisted last-successful-per-tier (drives oldest-first)
	inFlight *InFlight         // shared with the backup path — Scenario F
}

// SchedulerOptions configures a Scheduler.
type SchedulerOptions struct {
	Runner RestoreTestRunner
	Pick   CandidatePicker
	Store  *Store
	// Spec builds the run's spec (RestoreStorage, ScratchMin/Max, SourceTier, timeouts) from the
	// picked archive. Called ONCE PER RUN — see SpecBuilder for why it is not a value.
	Spec SpecBuilder
	// Cadence is the EVALUATION interval — how often due-ness is asked, NOT how often a test runs
	// (R-86). 0 → disabled.
	Cadence time.Duration
	// Settle is how long an archive must have sat before it is a restore-test candidate (R-86).
	// 0 → no settle requirement (any archive is a candidate).
	Settle time.Duration
	Logger *slog.Logger

	// R-85 (all optional — omit for the pre-R-85 single-tier behaviour):
	// Tiers are the configured tier target ids (primary first); TierPick resolves an archive on a
	// named tier; State persists last-successful-per-tier; InFlight is the shared one-heavy-op gate.
	Tiers    []string
	TierPick TierPicker
	State    *RestoreTestState
	InFlight *InFlight
}

// NewScheduler builds a Scheduler.
func NewScheduler(opts SchedulerOptions) *Scheduler {
	logger := opts.Logger
	if logger == nil {
		logger = slog.Default()
	}
	return &Scheduler{
		runner:   opts.Runner,
		pick:     opts.Pick,
		store:    opts.Store,
		spec:     opts.Spec,
		cadence:  opts.Cadence,
		settle:   opts.Settle,
		logger:   logger,
		now:      func() time.Time { return time.Now().UTC() },
		tiers:    append([]string(nil), opts.Tiers...),
		tierPick: opts.TierPick,
		rtState:  opts.State,
		inFlight: opts.InFlight,
	}
}

// Run EVALUATES due-ness on the interval until ctx is cancelled, and runs a restore-test only when
// a tier is actually due (R-86). A 0 interval disables it (the goroutine just waits for shutdown).
//
// The ticker survives as the evaluation interval and nothing else. It is emphatically NOT the
// trigger any more: its phase is the process's uptime, and agent deploys reset it, which is exactly
// the defect R-86 removes. What decides that a test happens is `EvaluateDue`.
//
// It still does NOT evaluate immediately on start — the first evaluation is one interval in. That
// is an EARNED restraint, kept deliberately: a restore is heavy, agent restarts are routine, and a
// crash-loop that evaluated at start would hammer a permanently-failing tier as fast as it could
// restart. Due-ness does not expire while we wait, so the only cost is up to one interval of
// latency on a tier that just became due. On-demand runs use `--selftest=restore-test`.
//
// Returns nil on ctx cancellation.
func (s *Scheduler) Run(ctx context.Context) error {
	if s.cadence <= 0 || s.runner == nil || s.spec == nil || (s.pick == nil && !s.rotating()) {
		s.logger.Info("backup: restore-test cadence disabled")
		<-ctx.Done()
		return nil
	}
	s.logger.Info("backup: restore-test scheduler starting (per-archive due-check)",
		"eval_interval", s.cadence, "settle", s.settle)
	t := time.NewTicker(s.cadence)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			s.logger.Info("backup: restore-test scheduler shutting down", "reason", ctx.Err())
			return nil
		case <-t.C:
			s.tick(ctx)
		}
	}
}

// tick is ONE EVALUATION: gate → due-check → run the first due tier → record which archive was
// proven. No-ops cleanly when nothing is due, when no backup exists yet, or when a heavy operation
// is already in flight. Deterministic given s.now — tests call it directly.
//
// One run per evaluation, by construction (Scenario F): a second due tier is left DUE and picked up
// by the next evaluation. Deferred, never cancelled, and never two multi-GB restores over one link.
func (s *Scheduler) tick(ctx context.Context) {
	if s.spec == nil {
		// Defensive: Run() already refuses to start without a SpecBuilder, but tick is also
		// reachable directly. Skipping loudly beats panicking the daemon goroutine — a missing
		// spec must cost a restore-test, never the agent.
		s.logger.Error("backup: restore-test has no spec builder — skipping (this is a wiring bug)")
		return
	}
	// The due-check runs BEFORE the gate is taken, and that ORDER is load-bearing under R-86.
	//
	// It used to be the other way round, and correctly so: the gate was held for one heavy run a
	// day, and the candidate lookup rode along inside it. Evaluations are now frequent, and the
	// lookup is a storage listing that for the offsite tier crosses the WAN. Holding the
	// one-heavy-operation gate for a read that answers "nothing to do" would open a small window at
	// EVERY evaluation in which a starting backup cannot acquire — and a backup that cannot acquire
	// does not merely wait, it records a failure and pages the operator (F-A1). A cheap poll must
	// not be able to manufacture that.
	//
	// Nothing is lost by checking first: due-ness does not expire, and the gate is still taken
	// before anything heavy begins.
	archive, target, err := s.pickForThisRun(ctx)
	if err != nil {
		s.logger.Warn("backup: restore-test could not pick a candidate; skipping", "err", err)
		return
	}
	if archive == "" {
		// A POSITIVE OBSERVABLE, at INFO, and this is not noise — it is standing rule 3.
		//
		// Before R-86 every tick ran a heavy restore-test, so the scheduler was audible by
		// construction. Now "nothing is due" is the NORMAL outcome, and at DEBUG it is silent: an
		// empty journal would be equally consistent with a healthy loop and with a dead goroutine,
		// which is the exact shape the R-88 watcher was retired for. One line per evaluation is four
		// lines a day at the 6h default, and it names each tier's verdict so the answer to "why did
		// nothing run last night?" is in the log rather than in a re-derivation.
		s.logger.Info("backup: restore-test evaluated — nothing due", "verdicts", s.verdictSummary(ctx))
		return
	}

	// Scenario F: join the one-heavy-operation-at-a-time gate. A restore-test PULLS a multi-GB
	// archive over the same tunnel an offsite backup PUSHES one; running both saturates the link and
	// drives each toward its timeout, which is how a healthy tier gets recorded as failed. DEFER —
	// never cancel what is already running: a deferred restore-test costs hours of coverage, a
	// cancelled backup costs the backup. A deferred tier stays DUE, so the next evaluation retries it.
	release, busy, ok := s.inFlight.TryAcquire("restore-test")
	if !ok {
		s.logger.Info("backup: restore-test deferred — a heavy operation is already in flight",
			"busy", busy, "target", target, "archive", archive)
		return
	}
	defer release()
	// R-85: build the spec for THIS run, from THIS archive. Never a frozen value.
	spec := s.spec(ctx, archive)
	spec.Archive = archive
	res := s.runner.RunRestoreTest(ctx, spec)
	if res.Skipped {
		return // already logged by the engine (no free scratch VMID)
	}
	rt := ToHubRestoreTest(res, s.now())
	s.store.RecordRestoreTest(rt)
	// Rotation credit is given ONLY on success. A failing tier must keep sorting first, or a tier
	// that fails every time would look freshly proven and quietly stop being retried.
	if rt.Pass && s.rtState != nil && target != "" {
		// R-86: the ARCHIVE is recorded, not merely the time — that is what makes the tier
		// not-due until a NEWER archive settles, and what makes a proof survive a restart.
		if err := s.rtState.RecordSuccess(target, archive, s.now()); err != nil {
			s.logger.Warn("backup: could not persist the restore-test proof state", "target", target, "err", err)
		}
	}
	switch {
	case !rt.Pass:
		// A failing restore-test is the loudest DR signal there is.
		s.logger.Error("backup: scheduled restore-test FAILED", "archive", rt.SourceArchive, "err", rt.Error)
	case len(res.StartWarnings) == 0:
		s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
	case res.WarningsRecognized:
		// Passed; the only warnings are the known-benign (e.g. systemd-nesting) advisory.
		s.logger.Info("backup: scheduled restore-test passed with warnings (recognized)",
			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
	default:
		// Passed liveness, but an UNRECOGNIZED start warning stood out — worth an operator look.
		s.logger.Warn("backup: scheduled restore-test passed with UNRECOGNIZED warnings",
			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
	}
}

// rotating reports whether multi-tier rotation is wired.
func (s *Scheduler) rotating() bool { return len(s.tiers) > 0 && s.tierPick != nil }

// pickForThisRun chooses the tier to test THIS evaluation: the first DUE tier, in oldest-proven
// order.
//
// R-86 changed what this answers. It used to answer "whose turn is it?", and the answer was always
// somebody's — the ticker had fired, so a test was going to happen. It now answers "is anything
// due?", and "nothing" is a normal, frequent and correct answer.
//
// OLDEST-FIRST (operator ruling 2026-07-26, Option 1) survives as the ORDER among due tiers: the
// tier whose last successful restore-test is oldest goes first, never-proven first of all. It is
// self-balancing, needs no config knob, and it still cannot starve a tier — but it no longer decides
// that a test happens at all.
//
// A tier with no settled archive is SKIPPED, not failed — a brand-new offsite tier has nothing to
// restore yet, and that is normal, not broken. A tier whose archives cannot be LISTED is likewise
// skipped, loudly, and its error is returned only when no other tier was testable: one tier's
// storage being unreadable must not cost the other tier its proof, and must not be silent either.
//
// Returns ("", "", nil) when nothing anywhere is due.
func (s *Scheduler) pickForThisRun(ctx context.Context) (archive, target string, err error) {
	if !s.rotating() {
		// Pre-R-85 single-tier path (tests and any caller that wires only `Pick`): there is no tier
		// identity and no persisted proof here, so there is nothing to compare an archive against
		// and no due-check is possible. It runs on every evaluation, exactly as it always did.
		a, perr := s.pick(ctx)
		return a, "", perr
	}
	var firstErr error
	for _, v := range s.EvaluateDue(ctx) {
		if v.Err != nil {
			s.logger.Warn("backup: restore-test candidate lookup failed for a tier; trying the next",
				"target", v.Target, "err", v.Err)
			if firstErr == nil {
				firstErr = v.Err
			}
			continue
		}
		if !v.Due {
			s.logger.Debug("backup: restore-test tier is not due", "target", v.Target, "reason", v.Reason)
			continue
		}
		s.logger.Info("backup: restore-test tier is DUE (per-archive; oldest-proven first among due tiers)",
			"target", v.Target, "archive", v.Archive, "landed", v.Landed.Format(time.RFC3339), "reason", v.Reason)
		return v.Archive, v.Target, nil
	}
	if firstErr != nil {
		return "", "", firstErr
	}
	return "", "", nil
}