7581f8140a
gates / gates (push) Successful in 7s
All three are the reporting and release path misreporting its own work. No
customer machine, no backup, no restore, no data. The restore-test itself and
when it runs are unchanged.
R-189 — a passing restore-test no longer vanishes on a restart. restore_tests[]
came only from the in-memory store, whose comment ("lost on restart; the cadence
re-populates") was true under a timer and stopped being true when R-86 made the
agent refuse to re-test a proven archive: the proof is then not repeated for a
whole archive generation. Observed live — a 14.5 GB offsite PASS reached no
host-report because the agent was restarted 2m43s later. RestoreTestState now
carries tier + verified beside the archive and renders reportable entries; the
collector merges them, one per tier, newest by TestedAt. It refuses to lie: a
record missing archive-or-tier produces no entry, and run mechanics are not
re-invented. Only successes are persisted, and the asymmetry is now written where
it will be read.
R-188 — a correct release stops emailing a failure. Only the tag PUSH moved
(build -> tag locally -> publish -> push tag): the push wakes CI, and a tag
visible before its package made the gate correctly fail a correct release about
half the time. The old order's invariant is asserted directly instead — the gate
now refuses a published version with no tag, as a bounded probe that prints its
own coverage, because the package listing api is still 401 without a token.
R-186 — a released binary can be verified by rebuilding it. -trimpath
-buildvcs=false: same source, same bytes, tag or no tag. Measured. publish-agent's
fallback also forced CGO_ENABLED=0 and produced a 74 KB different binary for the
same version; both paths now build identically. CLAUDE.md records the command.
594 lines
26 KiB
Go
594 lines
26 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
)
|
|
|
|
// R-86 — the restore-test follows the BACKUP, not the clock.
|
|
//
|
|
// Every test here DRIVES time (`s.now` is injected and stepped) rather than waiting for it. A test
|
|
// that slept could not say anything about a 24-hour rule in under 24 hours, and one that only
|
|
// asserted "no error" would pass against a scheduler that never ran anything at all — which is
|
|
// precisely the failure mode §8.1's trap produces. So the assertions are: did a test run, on WHICH
|
|
// archive, and did a second evaluation correctly run NOTHING.
|
|
|
|
// ── the fake tier storage ────────────────────────────────────────────────────────────────────
|
|
|
|
// archiveStub is one archive on a tier: its volid and when it landed.
|
|
type archiveStub struct {
|
|
volid string
|
|
landed time.Time
|
|
}
|
|
|
|
// tierStorage is a TierPicker over per-tier archive lists. It implements the SAME contract as the
|
|
// production picker (*BackupRunner).PickSettledRestoreCandidateOn — newest archive that landed at
|
|
// or before the cutoff — which is itself covered against a fake PVE API in backup_test.go, and
|
|
// end-to-end by the live run. Naming the seam explicitly: everything below is true up to this
|
|
// picker; that the real picker obeys the same rule is asserted there, not here.
|
|
type tierStorage struct {
|
|
archives map[string][]archiveStub
|
|
err map[string]error // target → lookup failure
|
|
}
|
|
|
|
func (ts *tierStorage) pick(_ context.Context, target string, notAfter time.Time) (string, time.Time, error) {
|
|
if e, ok := ts.err[target]; ok && e != nil {
|
|
return "", time.Time{}, e
|
|
}
|
|
var best archiveStub
|
|
for _, a := range ts.archives[target] {
|
|
if !notAfter.IsZero() && a.landed.After(notAfter) {
|
|
continue // not settled yet
|
|
}
|
|
if best.volid == "" || a.landed.After(best.landed) {
|
|
best = a
|
|
}
|
|
}
|
|
return best.volid, best.landed, nil
|
|
}
|
|
|
|
// dueHarness is a scheduler with a driven clock over a fake tier storage.
|
|
type dueHarness struct {
|
|
s *Scheduler
|
|
rr *rotRunner
|
|
st *RestoreTestState
|
|
ts *tierStorage
|
|
clock time.Time
|
|
path string
|
|
}
|
|
|
|
func newDueHarness(t *testing.T, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness {
|
|
t.Helper()
|
|
return newDueHarnessAt(t, filepath.Join(t.TempDir(), "rt.json"), start, settle, pass, tiers, ts)
|
|
}
|
|
|
|
func newDueHarnessAt(t *testing.T, statePath string, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness {
|
|
t.Helper()
|
|
h := &dueHarness{rr: &rotRunner{pass: pass}, ts: ts, clock: start, path: statePath}
|
|
h.st = NewRestoreTestState(statePath)
|
|
h.s = NewScheduler(SchedulerOptions{
|
|
Runner: h.rr,
|
|
Store: NewStore(),
|
|
Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
|
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009}
|
|
},
|
|
Cadence: time.Hour,
|
|
Settle: settle,
|
|
Logger: quiet(),
|
|
Tiers: tiers,
|
|
TierPick: ts.pick,
|
|
State: h.st,
|
|
InFlight: &InFlight{},
|
|
})
|
|
h.s.now = func() time.Time { return h.clock }
|
|
return h
|
|
}
|
|
|
|
// advance steps the clock by step, evaluating once at every step — the scheduler's real shape.
|
|
func (h *dueHarness) advance(step, total time.Duration) {
|
|
for elapsed := time.Duration(0); elapsed < total; elapsed += step {
|
|
h.clock = h.clock.Add(step)
|
|
h.s.tick(context.Background())
|
|
}
|
|
}
|
|
|
|
var day0 = time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC)
|
|
|
|
// dailyArchives lands one archive a day at 02:00 for n days, starting at day0.
|
|
func dailyArchives(tier string, n int) []archiveStub {
|
|
out := make([]archiveStub, 0, n)
|
|
for d := 0; d < n; d++ {
|
|
out = append(out, archiveStub{
|
|
volid: fmt.Sprintf("%s:backup/vzdump-lxc-9201-day%d.tar.zst", tier, d),
|
|
landed: day0.AddDate(0, 0, d),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ── SCENARIO A — a daily tier is proved daily, on its own archive ────────────────────────────
|
|
//
|
|
// THE TRAP THIS PINS (§8.1). R-86 reads "trigger a tier ~24 h after its own newest archive", and
|
|
// the literal implementation of that — *due when the newest archive is at least `settle` old* — is
|
|
// NEVER true on a daily tier: a new archive lands every day, so the newest archive's age resets to
|
|
// zero long before it reaches 24 h. The literal reading silently switches restore-testing OFF for
|
|
// the tier that matters most.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-08-03). In Scheduler.evaluateTier, the per-archive comparison
|
|
// was replaced by the naive age rule:
|
|
//
|
|
// - if ok && proven == archive { … not due … }
|
|
// - if s.now().Sub(landed) < s.settle { … not due … } // and the proven-archive check deleted
|
|
//
|
|
// and the picker cutoff was removed (`cutoff := time.Time{}`), i.e. exactly "is the newest archive
|
|
// old enough". Result:
|
|
//
|
|
// --- FAIL: TestDue_DailyTierIsProvedDailyOnItsOwnArchive
|
|
// restoretest_due_test.go: a daily tier must be proved once per day; got 0 run(s) over 5 days
|
|
//
|
|
// Zero runs — restore-testing off. Restored immediately afterwards.
|
|
func TestDue_DailyTierIsProvedDailyOnItsOwnArchive(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 6)}}
|
|
h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
|
|
|
// Five days, evaluated hourly.
|
|
h.advance(time.Hour, 5*24*time.Hour)
|
|
|
|
got := h.rr.seen()
|
|
if len(got) != 5 {
|
|
t.Fatalf("a daily tier must be proved once per day; got %d run(s) over 5 days: %v", len(got), got)
|
|
}
|
|
// And each run must be on the archive that settled that day — day0's on day 1, and so on.
|
|
for i, a := range got {
|
|
want := fmt.Sprintf("local:backup/vzdump-lxc-9201-day%d.tar.zst", i)
|
|
if a != want {
|
|
t.Fatalf("run %d tested %q, want %q — the test is not following the archive", i+1, a, want)
|
|
}
|
|
}
|
|
// The newest archive is NEVER the one tested: it has not settled.
|
|
if last := got[len(got)-1]; last == "local:backup/vzdump-lxc-9201-day5.tar.zst" {
|
|
t.Fatal("the still-settling archive was tested — the settle cutoff is not being applied")
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO B — a weekly tier is proved weekly, not every other day ─────────────────────────
|
|
func TestDue_WeeklyTierIsProvedOncePerArchive(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {
|
|
{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0},
|
|
{volid: "felhom-pbs:backup/ct/9201/w1", landed: day0.AddDate(0, 0, 7)},
|
|
{volid: "felhom-pbs:backup/ct/9201/w2", landed: day0.AddDate(0, 0, 14)},
|
|
}}}
|
|
h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
|
|
|
// Three weeks, evaluated every 6 hours — 84 evaluations.
|
|
h.advance(6*time.Hour, 21*24*time.Hour)
|
|
|
|
got := h.rr.seen()
|
|
want := []string{
|
|
"felhom-pbs:backup/ct/9201/w0",
|
|
"felhom-pbs:backup/ct/9201/w1",
|
|
"felhom-pbs:backup/ct/9201/w2",
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("a weekly tier must be proved ONCE PER ARCHIVE (3 archives over 3 weeks); got %d run(s): %v", len(got), got)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("run %d tested %q, want %q", i+1, got[i], want[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO C — an agent restart does not change the schedule ───────────────────────────────
|
|
//
|
|
// This is the defect a person actually notices: today every deploy restarts the ticker, so a
|
|
// restore-test runs one interval after each deploy regardless of what has already been proven.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-08-03): revert the state to per-tier TIME by making
|
|
// ProvenArchive ignore the stored archive —
|
|
//
|
|
// - if !ok || p.Archive == "" { return "", false }
|
|
// - return "", false // per-tier time only, the pre-R-86 state
|
|
//
|
|
// → --- FAIL: TestDue_RestartRunsNothing
|
|
//
|
|
// restoretest_due_test.go:226: an agent restart must not trigger a restore-test; 2 restart(s)
|
|
// produced 4 run(s)
|
|
//
|
|
// Four: the same already-proven archive re-tested on EVERY evaluation after EVERY restart, which is
|
|
// today's behaviour with the ticker's phase reset by the deploy. Restored.
|
|
func TestDue_RestartRunsNothing(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "rt.json")
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}}
|
|
start := day0.AddDate(0, 0, 1).Add(time.Hour) // day 1, 03:00 — day0's archive has settled
|
|
|
|
h := newDueHarnessAt(t, path, start, 24*time.Hour, true, []string{"local"}, ts)
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 1 {
|
|
t.Fatalf("precondition: the settled archive should have been proved once; got %d run(s)", n)
|
|
}
|
|
|
|
// --- two restarts: brand-new scheduler + brand-new state object over the SAME file ---
|
|
total := 0
|
|
for i := 0; i < 2; i++ {
|
|
h2 := newDueHarnessAt(t, path, start.Add(time.Duration(i+1)*time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
|
h2.s.tick(context.Background())
|
|
h2.s.tick(context.Background())
|
|
total += len(h2.rr.seen())
|
|
}
|
|
if total != 0 {
|
|
t.Fatalf("an agent restart must not trigger a restore-test; 2 restart(s) produced %d run(s)", total)
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO D — a new archive makes a tier due even if it was tested yesterday ──────────────
|
|
func TestDue_NewSettledArchiveMakesAProvedTierDueAgain(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
|
|
|
h.s.tick(context.Background()) // proves day0's archive
|
|
h.s.tick(context.Background()) // nothing new has settled → nothing
|
|
if n := len(h.rr.seen()); n != 1 {
|
|
t.Fatalf("want exactly 1 run before the new archive settles, got %d: %v", n, h.rr.seen())
|
|
}
|
|
|
|
// Day 2, 03:00 — day1's archive has now settled.
|
|
h.clock = day0.AddDate(0, 0, 2).Add(time.Hour)
|
|
h.s.tick(context.Background())
|
|
|
|
got := h.rr.seen()
|
|
if len(got) != 2 {
|
|
t.Fatalf("a newly settled archive must make the tier due again; got %v", got)
|
|
}
|
|
if got[1] != "local:backup/vzdump-lxc-9201-day1.tar.zst" {
|
|
t.Fatalf("the NEW archive must be the one tested; got %q", got[1])
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO E — a failing tier keeps being retried, and earns no proof ──────────────────────
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-08-03): give credit on failure in Scheduler.tick —
|
|
//
|
|
// - if rt.Pass && s.rtState != nil && target != "" {
|
|
// - if s.rtState != nil && target != "" {
|
|
//
|
|
// → --- FAIL: TestDue_FailingTierIsRetriedAndNeverProven
|
|
//
|
|
// restoretest_due_test.go: a failing tier must keep being retried; got 1 run(s) over 3
|
|
// evaluations
|
|
//
|
|
// A single failure would have retired the archive as proven — a permanently broken DR tier looking
|
|
// freshly verified, which is the loudest signal this system produces going silent. Restored.
|
|
func TestDue_FailingTierIsRetriedAndNeverProven(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 1)}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, false, []string{"local"}, ts)
|
|
|
|
for i := 0; i < 3; i++ {
|
|
h.s.tick(context.Background())
|
|
}
|
|
|
|
got := h.rr.seen()
|
|
if len(got) != 3 {
|
|
t.Fatalf("a failing tier must keep being retried; got %d run(s) over 3 evaluations: %v", len(got), got)
|
|
}
|
|
if _, ok := h.st.ProvenArchive("local"); ok {
|
|
t.Fatal("a FAILED restore-test must not record the archive as proven")
|
|
}
|
|
if _, ok := h.st.LastSuccess("local"); ok {
|
|
t.Fatal("a FAILED restore-test must not stamp the tier as proven")
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO F — two tiers due at once do not run at once ────────────────────────────────────
|
|
func TestDue_TwoDueTiersRunOneAtATime(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{
|
|
"local": {{volid: "local:backup/a.tar.zst", landed: day0}},
|
|
"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/a", landed: day0}},
|
|
}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
|
|
|
// Both tiers are due at this instant.
|
|
due := h.s.EvaluateDue(context.Background())
|
|
if len(due) != 2 || !due[0].Due || !due[1].Due {
|
|
t.Fatalf("precondition: both tiers should be due; got %v", due)
|
|
}
|
|
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 1 {
|
|
t.Fatalf("ONE evaluation must start ONE restore-test, never two multi-GB restores over one link; got %d: %v", n, h.rr.seen())
|
|
}
|
|
|
|
// The other tier was DEFERRED, not cancelled: it is still due and runs on the next evaluation.
|
|
h.s.tick(context.Background())
|
|
got := h.rr.seen()
|
|
if len(got) != 2 || got[0] == got[1] {
|
|
t.Fatalf("the deferred tier must run on the NEXT evaluation, on its own archive; got %v", got)
|
|
}
|
|
}
|
|
|
|
// The heavy-operation gate still holds, and a tier deferred behind a backup stays DUE.
|
|
func TestDue_DeferredBehindABackupStaysDue(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
|
|
|
gate := &InFlight{}
|
|
h.s.inFlight = gate
|
|
release, _, _ := gate.TryAcquire("backup:felhom-pbs")
|
|
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 0 {
|
|
t.Fatalf("the restore-test must DEFER while a backup holds the gate; got %d run(s)", n)
|
|
}
|
|
if due := h.s.EvaluateDue(context.Background()); !due[0].Due {
|
|
t.Fatal("a deferred tier must remain DUE — deferral is not dismissal")
|
|
}
|
|
release()
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 1 {
|
|
t.Fatalf("must resume once the gate frees; got %d run(s)", n)
|
|
}
|
|
}
|
|
|
|
// ── SCENARIO H — a newborn box is UNKNOWN, not stale and not a fault ─────────────────────────
|
|
func TestDue_NewbornTierIsNotDueAndNotAnError(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": nil}}
|
|
h := newDueHarness(t, day0, 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
|
|
|
due := h.s.EvaluateDue(context.Background())
|
|
if len(due) != 1 {
|
|
t.Fatalf("want one verdict, got %v", due)
|
|
}
|
|
v := due[0]
|
|
if v.Due || v.Err != nil || v.Archive != "" {
|
|
t.Fatalf("a tier with no archive is UNKNOWN — not due, not an error; got %+v", v)
|
|
}
|
|
if v.Reason == "" {
|
|
t.Fatal("every verdict must carry a reason — a due-check that cannot say why is a quiet path")
|
|
}
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 0 {
|
|
t.Fatalf("a newborn tier must not be restore-tested; got %d run(s)", n)
|
|
}
|
|
}
|
|
|
|
// An archive that exists but has NOT settled yet is not a candidate — and that is not an error.
|
|
func TestDue_UnsettledArchiveIsNotACandidate(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/fresh.tar.zst", landed: day0}}}}
|
|
h := newDueHarness(t, day0.Add(2*time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
|
|
|
if v := h.s.EvaluateDue(context.Background())[0]; v.Due || v.Archive != "" {
|
|
t.Fatalf("an archive 2h old must not be a candidate under a 24h settle lag; got %+v", v)
|
|
}
|
|
h.s.tick(context.Background())
|
|
if n := len(h.rr.seen()); n != 0 {
|
|
t.Fatalf("nothing settled → no run; got %d", n)
|
|
}
|
|
}
|
|
|
|
// A tier whose archives cannot be LISTED is UNKNOWN — never silently "not due", and never silent.
|
|
// Treating a lookup failure as "not due" would retire a tier the moment its storage stopped
|
|
// answering, which is the same absence-is-not-evidence error this monitor family keeps making.
|
|
func TestDue_LookupFailureIsUnknownNotNotDue(t *testing.T) {
|
|
boom := errors.New("storage unreachable")
|
|
ts := &tierStorage{
|
|
archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}},
|
|
err: map[string]error{"felhom-pbs": boom},
|
|
}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
|
|
|
var pbs DueVerdict
|
|
for _, v := range h.s.EvaluateDue(context.Background()) {
|
|
if v.Target == "felhom-pbs" {
|
|
pbs = v
|
|
}
|
|
}
|
|
if pbs.Err == nil {
|
|
t.Fatal("a lookup failure must travel in the verdict, not be swallowed")
|
|
}
|
|
if pbs.Due {
|
|
t.Fatal("a tier we could not list must not be reported DUE — we have no archive to test")
|
|
}
|
|
if pbs.Reason == "" {
|
|
t.Fatal("the failure must be explained, not merely flagged")
|
|
}
|
|
|
|
// And the OTHER tier still runs: one tier's storage being unreadable must not cost the other
|
|
// tier its proof.
|
|
h.s.tick(context.Background())
|
|
if got := h.rr.seen(); len(got) != 1 || got[0] != "local:backup/a.tar.zst" {
|
|
t.Fatalf("the readable tier must still be proved; got %v", got)
|
|
}
|
|
}
|
|
|
|
// ── the state's migration (§8.2) ─────────────────────────────────────────────────────────────
|
|
|
|
// A pre-R-86 state file carries a TIME and no archive. It must keep its time (rotation ordering
|
|
// survives the upgrade) and yield NO proven archive, so each tier is due exactly once. Reading a
|
|
// legacy time as proof of the CURRENT archive would mark an unproven archive proven — a guarantee
|
|
// invented by a migration.
|
|
func TestRestoreTestState_LegacyFileMigratesToNothingProven(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "rt.json")
|
|
legacy := `{"local":"2026-08-01T02:00:00Z","felhom-pbs":"2026-07-30T02:00:00Z"}`
|
|
if err := writeFileForTest(path, legacy); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
st := NewRestoreTestState(path)
|
|
if _, ok := st.ProvenArchive("local"); ok {
|
|
t.Fatal("a legacy record names no archive — it must NOT be read as proof of the current one")
|
|
}
|
|
at, ok := st.LastSuccess("local")
|
|
if !ok || !at.Equal(time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC)) {
|
|
t.Fatalf("the legacy TIME must survive (rotation ordering depends on it); got %v ok=%v", at, ok)
|
|
}
|
|
// Ordering still works off the legacy times.
|
|
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
|
|
t.Fatalf("oldest-first must still order legacy records; got %v", got)
|
|
}
|
|
}
|
|
|
|
// The new shape round-trips, archive and all.
|
|
func TestRestoreTestState_ArchiveRoundTrips(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "rt.json")
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
st := NewRestoreTestState(path)
|
|
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
re := NewRestoreTestState(path)
|
|
got, ok := re.ProvenArchive("felhom-pbs")
|
|
if !ok || got != "felhom-pbs:backup/ct/9201/x" {
|
|
t.Fatalf("the proven ARCHIVE must survive a restart; got %q ok=%v", got, ok)
|
|
}
|
|
at, ok := re.LastSuccess("felhom-pbs")
|
|
if !ok || !at.Equal(now) {
|
|
t.Fatalf("the proven TIME must survive too; got %v ok=%v", at, ok)
|
|
}
|
|
}
|
|
|
|
// writeFileForTest is a tiny helper so the legacy-migration fixture reads clearly above.
|
|
func writeFileForTest(path, content string) error {
|
|
return os.WriteFile(path, []byte(content), 0o600)
|
|
}
|
|
|
|
// Standing rule 3: an absent log line is not evidence. "Nothing is due" is now the NORMAL outcome of
|
|
// an evaluation, so it must produce a POSITIVE observable naming each tier's verdict — otherwise a
|
|
// quiet journal is equally consistent with a healthy loop and a dead goroutine.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-08-03): drop the summary back to a bare
|
|
// `s.logger.Debug("backup: restore-test not due this evaluation")` and this fails with
|
|
// "a not-due evaluation must name each tier's verdict; got \"\"" — i.e. nothing at INFO at all.
|
|
func TestDue_NothingDueStillNamesEveryTiersVerdict(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{
|
|
"local": {{volid: "local:backup/a.tar.zst", landed: day0}},
|
|
"felhom-pbs": nil, // no archive at all
|
|
}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
|
// Prove the local tier so NOTHING is due.
|
|
if err := h.st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", h.clock); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Assert what the SCHEDULER emits on a real evaluation, not what a helper returns — a helper
|
|
// test would pass against a tick that never calls it.
|
|
var logbuf strings.Builder
|
|
h.s.logger = slog.New(slog.NewTextHandler(&logbuf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
|
h.s.tick(context.Background())
|
|
got := logbuf.String()
|
|
for _, want := range []string{"local", "felhom-pbs", "already proven", "no settled archive"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Fatalf("a not-due evaluation must name each tier's verdict; got %q (missing %q)", got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A tier whose storage cannot be listed must say UNKNOWN in that same line — a lookup failure that
|
|
// reads as "nothing due" is the silence this rule exists to prevent.
|
|
func TestDue_VerdictSummaryNamesAnUnknownTier(t *testing.T) {
|
|
ts := &tierStorage{
|
|
archives: map[string][]archiveStub{"local": nil},
|
|
err: map[string]error{"felhom-pbs": errors.New("storage unreachable")},
|
|
}
|
|
h := newDueHarness(t, day0, 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
|
got := h.s.verdictSummary(context.Background())
|
|
if !strings.Contains(got, "UNKNOWN") || !strings.Contains(got, "storage unreachable") {
|
|
t.Fatalf("an unlistable tier must read as UNKNOWN with its error; got %q", got)
|
|
}
|
|
}
|
|
|
|
// ── R-189 — the persisted proof must be REPORTABLE, and must refuse to lie ───────────────────
|
|
//
|
|
// A proof held only in the in-memory store dies with the process, and under per-archive due-ness the
|
|
// agent will not repeat the work. So the persisted record has to be able to become a host-report
|
|
// entry — without inventing anything it does not know.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-08-03): drop the `reportable()` filter from
|
|
// ProvenRestoreTests, so a pre-R-189 record (archive but no tier) is emitted →
|
|
//
|
|
// --- FAIL: TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe
|
|
// restoretest_due_test.go: a record with no TIER must not be reported (the hub keys its
|
|
// per-tier proof on it); got [{... SourceTier: ...}]
|
|
//
|
|
// Restored.
|
|
func TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "rt.json")
|
|
// v1 (a bare time), v2 (archive, no tier) and v3 (complete) side by side — every shape this
|
|
// file has ever had, which is what a real box carries after two upgrades.
|
|
legacy := `{
|
|
"old-v1": "2026-07-30T02:11:07Z",
|
|
"old-v2": {"archive":"felhom-backup:backup/vzdump-lxc-9201-a.tar.zst","proven_at":"2026-08-01T04:41:58Z"},
|
|
"felhom-pbs": {"archive":"felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z","tier":"pbs","verified":"boot+running","proven_at":"2026-08-03T13:25:14Z"}
|
|
}`
|
|
if err := writeFileForTest(path, legacy); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got := NewRestoreTestState(path).ProvenRestoreTests(context.Background())
|
|
if len(got) != 1 {
|
|
t.Fatalf("only the record that can be described honestly may be reported; got %d: %+v", len(got), got)
|
|
}
|
|
e := got[0]
|
|
if e.SourceTier != "pbs" {
|
|
t.Fatalf("a record with no TIER must not be reported (the hub keys its per-tier proof on it); got %+v", got)
|
|
}
|
|
if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" || !e.Pass {
|
|
t.Fatalf("the reported entry must be the stored proof, unchanged; got %+v", e)
|
|
}
|
|
if e.TestedAt != "2026-08-03T13:25:14Z" {
|
|
t.Fatalf("the entry must carry the time the run passed, not now(); got %q", e.TestedAt)
|
|
}
|
|
if e.Verified != "boot+running" {
|
|
t.Fatalf("what the run verified must survive the round trip; got %q", e.Verified)
|
|
}
|
|
// Run mechanics are NOT invented: an absent duration is not a claim, a fabricated one would be.
|
|
if e.DurationSeconds != 0 || e.ScratchVMID != 0 {
|
|
t.Fatalf("the re-report must not invent run mechanics it never stored; got duration=%v scratch=%d",
|
|
e.DurationSeconds, e.ScratchVMID)
|
|
}
|
|
// The legacy records still serve the DUE-check, which is a separate question from reporting.
|
|
if _, ok := NewRestoreTestState(path).ProvenArchive("old-v2"); !ok {
|
|
t.Fatal("a v2 record must still answer the due-check even though it cannot be reported")
|
|
}
|
|
}
|
|
|
|
// A tier proved through the SCHEDULER (not by hand) lands in the state complete enough to report —
|
|
// the production path, not a hand-built fixture.
|
|
func TestScheduler_ProofIsRecordedReportably(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
|
// The fake runner echoes the spec's tier; give the spec a tier the way main.go does.
|
|
h.s.spec = func(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
|
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: "pbs"}
|
|
}
|
|
h.s.tick(context.Background())
|
|
|
|
got := h.st.ProvenRestoreTests(context.Background())
|
|
if len(got) != 1 {
|
|
t.Fatalf("a scheduled pass must leave a REPORTABLE proof; got %d: %+v", len(got), got)
|
|
}
|
|
if got[0].SourceTier != "pbs" || got[0].SourceArchive != "felhom-pbs:backup/ct/9201/w0" {
|
|
t.Fatalf("the proof must name the tier and the archive the run used; got %+v", got[0])
|
|
}
|
|
}
|
|
|
|
// A FAILED run leaves nothing to report — the asymmetry of §8.1, asserted rather than assumed.
|
|
func TestScheduler_AFailureLeavesNoPersistedProof(t *testing.T) {
|
|
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
|
|
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, false, []string{"felhom-pbs"}, ts)
|
|
h.s.tick(context.Background())
|
|
if got := h.st.ProvenRestoreTests(context.Background()); len(got) != 0 {
|
|
t.Fatalf("a FAILED run must persist nothing — a failing tier is retried, and a stored failure "+
|
|
"would outlive the fault; got %+v", got)
|
|
}
|
|
}
|