765d8b3168
SchedulerOptions.Spec was a VALUE produced by an immediately-invoked function at daemon start, so storageTier() and restoreTaskTimeout() were evaluated once and reused for every run for the process lifetime. Nothing tier-varying was expressible (the offsite tier could never be scheduled), and it was a latent staleness bug besides: a storage-type or config change did not take effect until restart. - backup.SpecBuilder: func(ctx, archive) RestoreTestSpec, called once per run. The archive is passed because the tier MUST come from it (v0.100.0 rule) — config-derived is what classified a PBS archive as 'local' and killed a 14.46 GB WAN restore at the 10-minute local bound. - A nil spec builder SKIPS loudly instead of panicking: a wiring bug must cost a restore-test, never the daemon goroutine. Red-proof observed. Full suite green (29 packages, rc=0).
125 lines
4.0 KiB
Go
125 lines
4.0 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
)
|
|
|
|
// R-85 (1.1) — the spec is built PER RUN, never frozen at construction.
|
|
//
|
|
// It used to be an immediately-invoked function at daemon start, so storageTier() and
|
|
// restoreTaskTimeout() were evaluated ONCE and the value reused for every run for the process
|
|
// lifetime. That is what made an offsite restore-test impossible to schedule at all, and it was a
|
|
// latent staleness bug besides: a storage-type or config change did not take effect until restart.
|
|
|
|
type specSpy struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
archives []string
|
|
tiers []string // what the builder decided, per call
|
|
}
|
|
|
|
func (sp *specSpy) build(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
|
sp.mu.Lock()
|
|
defer sp.mu.Unlock()
|
|
sp.calls++
|
|
sp.archives = append(sp.archives, archive)
|
|
// Decide the tier from the ARCHIVE, exactly as main.go does (the v0.100.0 rule).
|
|
tier := "local"
|
|
if len(archive) > 10 && archive[:10] == "felhom-pbs" {
|
|
tier = "pbs"
|
|
}
|
|
sp.tiers = append(sp.tiers, tier)
|
|
return reconcile.RestoreTestSpec{
|
|
RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: tier,
|
|
}
|
|
}
|
|
|
|
// COMPANION RED-PROOF (observed): change Scheduler.spec back to a frozen
|
|
// `reconcile.RestoreTestSpec` value captured at construction → this fails with
|
|
// "the spec builder must run ONCE PER RUN, got 1 call(s) across 3 ticks", because a frozen value is
|
|
// evaluated exactly once no matter how many ticks fire. Restored.
|
|
func TestScheduler_SpecIsBuiltPerRun(t *testing.T) {
|
|
sp := &specSpy{}
|
|
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
|
|
n := 0
|
|
s := NewScheduler(SchedulerOptions{
|
|
Runner: rt,
|
|
Pick: func(context.Context) (string, error) {
|
|
n++
|
|
return fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", n), nil
|
|
},
|
|
Store: NewStore(),
|
|
Spec: sp.build,
|
|
Cadence: time.Hour,
|
|
Logger: quiet(),
|
|
})
|
|
|
|
for i := 0; i < 3; i++ {
|
|
s.tick(context.Background())
|
|
}
|
|
|
|
sp.mu.Lock()
|
|
defer sp.mu.Unlock()
|
|
if sp.calls != 3 {
|
|
t.Fatalf("the spec builder must run ONCE PER RUN, got %d call(s) across 3 ticks", sp.calls)
|
|
}
|
|
// And it must see the archive THIS run picked — not a stale one.
|
|
for i, a := range sp.archives {
|
|
want := fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", i+1)
|
|
if a != want {
|
|
t.Fatalf("run %d: builder saw archive %q, want %q — the spec is not tracking the picked archive", i+1, a, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The tier must follow the ARCHIVE across runs. A builder that saw only the configured target would
|
|
// return the same tier every time — which is exactly the v0.100.0 defect that killed a 14.46 GB WAN
|
|
// restore at the 10-minute local bound.
|
|
func TestScheduler_SpecTierFollowsTheArchive(t *testing.T) {
|
|
sp := &specSpy{}
|
|
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
|
|
archives := []string{
|
|
"local:backup/vzdump-lxc-9201-x.tar.zst",
|
|
"felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z",
|
|
}
|
|
i := 0
|
|
s := NewScheduler(SchedulerOptions{
|
|
Runner: rt,
|
|
Pick: func(context.Context) (string, error) {
|
|
a := archives[i%len(archives)]
|
|
i++
|
|
return a, nil
|
|
},
|
|
Store: NewStore(), Spec: sp.build, Cadence: time.Hour, Logger: quiet(),
|
|
})
|
|
s.tick(context.Background())
|
|
s.tick(context.Background())
|
|
|
|
sp.mu.Lock()
|
|
defer sp.mu.Unlock()
|
|
if len(sp.tiers) != 2 || sp.tiers[0] != "local" || sp.tiers[1] != "pbs" {
|
|
t.Fatalf("the tier must follow the archive per run; got %v", sp.tiers)
|
|
}
|
|
}
|
|
|
|
// A nil spec builder must SKIP loudly, not panic — a wiring bug costs a restore-test, never the
|
|
// daemon goroutine.
|
|
func TestScheduler_NilSpecSkipsInsteadOfPanicking(t *testing.T) {
|
|
rt := &fakeRTRunner{}
|
|
s := NewScheduler(SchedulerOptions{
|
|
Runner: rt,
|
|
Pick: func(context.Context) (string, error) { return "vol", nil },
|
|
Store: NewStore(), Cadence: time.Hour, Logger: quiet(),
|
|
})
|
|
s.tick(context.Background()) // must not panic
|
|
if rt.runs != 0 {
|
|
t.Fatalf("a nil spec must not run a restore-test; got %d run(s)", rt.runs)
|
|
}
|
|
}
|