R-85 Phase 1: the restore-test spec is built PER RUN, not frozen at daemon start

SchedulerOptions.Spec was a VALUE produced by an immediately-invoked function at daemon start, so storageTier() and restoreTaskTimeout() were evaluated once and reused for every run for the process lifetime. Nothing tier-varying was expressible (the offsite tier could never be scheduled), and it was a latent staleness bug besides: a storage-type or config change did not take effect until restart. - backup.SpecBuilder: func(ctx, archive) RestoreTestSpec, called once per run. The archive is passed because the tier MUST come from it (v0.100.0 rule) — config-derived is what classified a PBS archive as 'local' and killed a 14.46 GB WAN restore at the 10-minute local bound. - A nil spec builder SKIPS loudly instead of panicking: a wiring bug must cost a restore-test, never the daemon goroutine. Red-proof observed. Full suite green (29 packages, rc=0).
2026-07-26 20:47:28 +02:00
parent edde8a01ca
commit 765d8b3168
5 changed files with 204 additions and 14 deletions
@@ -0,0 +1,124 @@
+package backup
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
+)
+
+// R-85 (1.1) — the spec is built PER RUN, never frozen at construction.
+//
+// It used to be an immediately-invoked function at daemon start, so storageTier() and
+// restoreTaskTimeout() were evaluated ONCE and the value reused for every run for the process
+// lifetime. That is what made an offsite restore-test impossible to schedule at all, and it was a
+// latent staleness bug besides: a storage-type or config change did not take effect until restart.
+
+type specSpy struct {
+	mu       sync.Mutex
+	calls    int
+	archives []string
+	tiers    []string // what the builder decided, per call
+}
+
+func (sp *specSpy) build(_ context.Context, archive string) reconcile.RestoreTestSpec {
+	sp.mu.Lock()
+	defer sp.mu.Unlock()
+	sp.calls++
+	sp.archives = append(sp.archives, archive)
+	// Decide the tier from the ARCHIVE, exactly as main.go does (the v0.100.0 rule).
+	tier := "local"
+	if len(archive) > 10 && archive[:10] == "felhom-pbs" {
+		tier = "pbs"
+	}
+	sp.tiers = append(sp.tiers, tier)
+	return reconcile.RestoreTestSpec{
+		RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: tier,
+	}
+}
+
+// COMPANION RED-PROOF (observed): change Scheduler.spec back to a frozen
+// `reconcile.RestoreTestSpec` value captured at construction → this fails with
+// "the spec builder must run ONCE PER RUN, got 1 call(s) across 3 ticks", because a frozen value is
+// evaluated exactly once no matter how many ticks fire. Restored.
+func TestScheduler_SpecIsBuiltPerRun(t *testing.T) {
+	sp := &specSpy{}
+	rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
+	n := 0
+	s := NewScheduler(SchedulerOptions{
+		Runner: rt,
+		Pick: func(context.Context) (string, error) {
+			n++
+			return fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", n), nil
+		},
+		Store:   NewStore(),
+		Spec:    sp.build,
+		Cadence: time.Hour,
+		Logger:  quiet(),
+	})
+
+	for i := 0; i < 3; i++ {
+		s.tick(context.Background())
+	}
+
+	sp.mu.Lock()
+	defer sp.mu.Unlock()
+	if sp.calls != 3 {
+		t.Fatalf("the spec builder must run ONCE PER RUN, got %d call(s) across 3 ticks", sp.calls)
+	}
+	// And it must see the archive THIS run picked — not a stale one.
+	for i, a := range sp.archives {
+		want := fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", i+1)
+		if a != want {
+			t.Fatalf("run %d: builder saw archive %q, want %q — the spec is not tracking the picked archive", i+1, a, want)
+		}
+	}
+}
+
+// The tier must follow the ARCHIVE across runs. A builder that saw only the configured target would
+// return the same tier every time — which is exactly the v0.100.0 defect that killed a 14.46 GB WAN
+// restore at the 10-minute local bound.
+func TestScheduler_SpecTierFollowsTheArchive(t *testing.T) {
+	sp := &specSpy{}
+	rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
+	archives := []string{
+		"local:backup/vzdump-lxc-9201-x.tar.zst",
+		"felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z",
+	}
+	i := 0
+	s := NewScheduler(SchedulerOptions{
+		Runner: rt,
+		Pick: func(context.Context) (string, error) {
+			a := archives[i%len(archives)]
+			i++
+			return a, nil
+		},
+		Store: NewStore(), Spec: sp.build, Cadence: time.Hour, Logger: quiet(),
+	})
+	s.tick(context.Background())
+	s.tick(context.Background())
+
+	sp.mu.Lock()
+	defer sp.mu.Unlock()
+	if len(sp.tiers) != 2 || sp.tiers[0] != "local" || sp.tiers[1] != "pbs" {
+		t.Fatalf("the tier must follow the archive per run; got %v", sp.tiers)
+	}
+}
+
+// A nil spec builder must SKIP loudly, not panic — a wiring bug costs a restore-test, never the
+// daemon goroutine.
+func TestScheduler_NilSpecSkipsInsteadOfPanicking(t *testing.T) {
+	rt := &fakeRTRunner{}
+	s := NewScheduler(SchedulerOptions{
+		Runner: rt,
+		Pick:   func(context.Context) (string, error) { return "vol", nil },
+		Store:  NewStore(), Cadence: time.Hour, Logger: quiet(),
+	})
+	s.tick(context.Background()) // must not panic
+	if rt.runs != 0 {
+		t.Fatalf("a nil spec must not run a restore-test; got %d run(s)", rt.runs)
+	}
+}