felhom-agent/internal/backup/schedule.go

package backup

import (
	"context"
	"log/slog"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
)

// RestoreTestRunner is the reconcile-engine seam the scheduler drives (*reconcile.Engine
// satisfies it). Kept narrow so the scheduler is unit-testable with a fake.
type RestoreTestRunner interface {
	RunRestoreTest(ctx context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult
}

// CandidatePicker resolves the archive volid to restore-test (newest backup), or "" when
// there is none yet (the tick then no-ops).
type CandidatePicker func(ctx context.Context) (string, error)

// Scheduler runs the self-restore-test on an agent-internal cadence. It is the fourth daemon
// goroutine; it does real restore→boot→destroy, so it only runs when the cadence is enabled
// AND a valid scratch band is configured (validated by the caller before construction).
type Scheduler struct {
	runner  RestoreTestRunner
	pick    CandidatePicker
	store   *Store
	spec    reconcile.RestoreTestSpec // archive is filled per-tick
	cadence time.Duration
	logger  *slog.Logger
	now     func() time.Time
}

// SchedulerOptions configures a Scheduler.
type SchedulerOptions struct {
	Runner  RestoreTestRunner
	Pick    CandidatePicker
	Store   *Store
	Spec    reconcile.RestoreTestSpec // RestoreStorage, ScratchMin/Max, SourceTier, BootTimeout
	Cadence time.Duration             // 0 → disabled
	Logger  *slog.Logger
}

// NewScheduler builds a Scheduler.
func NewScheduler(opts SchedulerOptions) *Scheduler {
	logger := opts.Logger
	if logger == nil {
		logger = slog.Default()
	}
	return &Scheduler{
		runner:  opts.Runner,
		pick:    opts.Pick,
		store:   opts.Store,
		spec:    opts.Spec,
		cadence: opts.Cadence,
		logger:  logger,
		now:     func() time.Time { return time.Now().UTC() },
	}
}

// Run fires a restore-test on the cadence until ctx is cancelled. A 0 cadence disables it
// (the goroutine just waits for shutdown). It does NOT fire immediately on start (a restore
// is heavy; the first runs one interval in) — on-demand runs use the selftest harness.
// Returns nil on ctx cancellation.
func (s *Scheduler) Run(ctx context.Context) error {
	if s.cadence <= 0 || s.runner == nil || s.pick == nil {
		s.logger.Info("backup: restore-test cadence disabled")
		<-ctx.Done()
		return nil
	}
	s.logger.Info("backup: restore-test scheduler starting", "cadence", s.cadence)
	t := time.NewTicker(s.cadence)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			s.logger.Info("backup: restore-test scheduler shutting down", "reason", ctx.Err())
			return nil
		case <-t.C:
			s.tick(ctx)
		}
	}
}

// tick runs one scheduled restore-test: pick a backup → run → record. No-ops cleanly when
// no backup exists yet. Deterministic given s.now — tests call it directly.
func (s *Scheduler) tick(ctx context.Context) {
	archive, err := s.pick(ctx)
	if err != nil {
		s.logger.Warn("backup: restore-test could not pick a candidate; skipping", "err", err)
		return
	}
	if archive == "" {
		s.logger.Info("backup: restore-test skipped; no backup available yet")
		return
	}
	spec := s.spec
	spec.Archive = archive
	res := s.runner.RunRestoreTest(ctx, spec)
	if res.Skipped {
		return // already logged by the engine (no free scratch VMID)
	}
	rt := ToHubRestoreTest(res, s.now())
	s.store.RecordRestoreTest(rt)
	switch {
	case !rt.Pass:
		// A failing restore-test is the loudest DR signal there is.
		s.logger.Error("backup: scheduled restore-test FAILED", "archive", rt.SourceArchive, "err", rt.Error)
	case len(res.StartWarnings) == 0:
		s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
	case res.WarningsRecognized:
		// Passed; the only warnings are the known-benign (e.g. systemd-nesting) advisory.
		s.logger.Info("backup: scheduled restore-test passed with warnings (recognized)",
			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
	default:
		// Passed liveness, but an UNRECOGNIZED start warning stood out — worth an operator look.
		s.logger.Warn("backup: scheduled restore-test passed with UNRECOGNIZED warnings",
			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
	}
}