package localapi import ( "context" "encoding/json" "errors" "io" "log/slog" "net/http" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // R-82 Slice A — per-target cadence, due and runner. // // The agent and the controller deploy INDEPENDENTLY. A new agent will serve old controllers for as // long as it takes the fleet to catch up, so the untargeted contract is frozen, not merely // "probably fine". These tests pin that freeze; the multi-tier behaviour is additive on top. // tieredServer builds a two-tier server: primary "local" (24h) + "felhom-pbs" (7d), each with its // own runner, exactly as main.go wires it. func tieredServer(t *testing.T, st *fakeStore, localSvc, pbsSvc *fakeBackups) *Server { t.Helper() srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: localSvc, Store: st, // Both tier targets must be PRESENT in the storage view: since v0.102.0 a tier whose target // storage is absent DEFERS. A real box has both; a fake with no targets would silently // defer every tier and make these assertions vacuous. Storage: fakeStorage{targets: []hub.StorageTarget{ {Name: "local", Type: "local"}, {Name: "felhom-pbs", Type: "pbs"}, }}, Tokens: staticTokens{"A": 8200, "B": 9300}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: localSvc}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc}, }, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatalf("new server: %v", err) } srv.baseCtx = context.Background() srv.now = func() time.Time { return testNow } return srv } // seen returns the vmids this fake runner was invoked for (mutex-guarded — the backup runs on a // goroutine). func (f *fakeBackups) seen() []int { f.mu.Lock() defer f.mu.Unlock() return append([]int(nil), f.vmids...) } // waitFor polls cond for up to 2s. POST /backup is fire-and-forget, so the assertion has to wait // for the goroutine rather than assume it has run. func waitFor(t *testing.T, cond func() bool) { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if cond() { return } time.Sleep(5 * time.Millisecond) } t.Fatalf("condition not met within 2s") } func backupAt(target string, vmid int, ago time.Duration, ok bool) hub.Backup { return hub.Backup{ TargetID: target, VMID: vmid, Success: ok, StartedAt: testNow.Add(-ago).Format(time.RFC3339), } } // ── RED-PROOF 1 — old controller ↔ new agent ──────────────────────────────────────────────── // // An old controller sends `GET /backup/due` with no query string and parses the pre-R-82 response. // The response must be BYTE-IDENTICAL — not merely semantically similar. A stray `"target":"local"` // key is harmless to a tolerant JSON decoder and fatal to a strict one, and we do not get to choose // which the deployed fleet has. // // COMPANION RED-PROOF (observed): drop the `omitempty` from BackupDueResponse.Target and have // tierFromRequest echo the primary's id for an untargeted request → this test fails with the // observed body carrying `"target":"local"`. Restored. func TestBackupDue_Untargeted_ResponseBytesUnchanged(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true)) h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() rr := do(t, h, "GET", "/backup/due", "A", "") if rr.Code != http.StatusOK { t.Fatalf("status = %d, want 200", rr.Code) } var got map[string]any if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil { t.Fatalf("unmarshal: %v (body %s)", err, rr.Body.String()) } data, _ := got["data"].(map[string]any) if data == nil { t.Fatalf("no data object in %s", rr.Body.String()) } if _, present := data["target"]; present { t.Fatalf("UNTARGETED response MUST NOT carry a target key — an old controller sees a changed contract; body: %s", rr.Body.String()) } if data["due"] != false { t.Fatalf("2h-old local backup under a 24h cadence must not be due; body: %s", rr.Body.String()) } if data["reason"] != "within cadence window" { t.Fatalf("reason string changed: %v", data["reason"]) } } // The untargeted verdict must come from the PRIMARY tier's cadence, not from whichever tier // happens to be freshest. With a stale local and a fresh PBS backup, untargeted must say DUE. func TestBackupDue_Untargeted_UsesPrimaryTierOnly(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("local", 8200, 30*time.Hour, true)) // stale for 24h cadence st.RecordBackup(backupAt("felhom-pbs", 8200, 1*time.Hour, true)) // fresh, different tier h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() var resp struct { Data BackupDueResponse `json:"data"` } rr := do(t, h, "GET", "/backup/due", "A", "") if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { t.Fatal(err) } if !resp.Data.Due { t.Fatalf("a fresh backup on ANOTHER tier must not satisfy the primary's cadence; got %+v", resp.Data) } } // ── Per-tier due-ness ──────────────────────────────────────────────────────────────────────── // THE POINT OF THE WHOLE SLICE: a fresh daily local backup must NOT satisfy the weekly PBS tier. // Without the per-target filter in latestSuccessfulBackupForTarget the DR tier would never run — // which is exactly today's "applied and empty" state, re-created in code. func TestBackupDue_PerTier_LocalFreshDoesNotSatisfyPBS(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true)) // fresh daily st.RecordBackup(backupAt("felhom-pbs", 8200, 8*24*time.Hour, true)) // 8d — past the 7d weekly h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() var local, pbs struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil { t.Fatal(err) } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if local.Data.Due { t.Fatalf("local tier: 2h old under 24h cadence must NOT be due; got %+v", local.Data) } if !pbs.Data.Due { t.Fatalf("PBS tier: 8d old under a 7d cadence MUST be due; got %+v", pbs.Data) } if pbs.Data.Target != "felhom-pbs" || local.Data.Target != "local" { t.Fatalf("a targeted response must echo its tier; got local=%q pbs=%q", local.Data.Target, pbs.Data.Target) } } // A 6-day-old PBS snapshot is INSIDE the weekly window — it must not be due. The mirror of the // hub-side threshold test in Slice C, asserted here at the source of truth. func TestBackupDue_PerTier_PBSWithinWeeklyWindow(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("felhom-pbs", 8200, 6*24*time.Hour, true)) h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() var pbs struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if pbs.Data.Due { t.Fatalf("6d old under a 7d cadence must NOT be due; got %+v", pbs.Data) } } // A failed backup must not satisfy any tier's cadence (pre-existing rule, re-asserted per-tier). func TestBackupDue_PerTier_FailedBackupDoesNotCount(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, false)) h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() var pbs struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if !pbs.Data.Due { t.Fatalf("a FAILED backup must not satisfy the cadence; got %+v", pbs.Data) } } // The fail-safe-toward-due rule survives per-tier: an unparseable timestamp yields DUE. // A spurious backup is cheap; a skipped one is not. func TestBackupDue_PerTier_UnparseableTimestampIsDue(t *testing.T) { st := &fakeStore{} st.RecordBackup(hub.Backup{TargetID: "felhom-pbs", VMID: 8200, Success: true, StartedAt: "not-a-time"}) h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() var pbs struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if !pbs.Data.Due { t.Fatalf("unparseable timestamp must fail SAFE toward due; got %+v", pbs.Data) } } // An unknown target is a 400 — never a silent fallback to the primary. A controller asking about a // tier this agent does not serve must find out, not be handed a different tier's freshness and act // on it. func TestBackupDue_UnknownTarget_IsAnErrorNotAFallback(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true)) h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler() rr := do(t, h, "GET", "/backup/due?target=does-not-exist", "A", "") if rr.Code != http.StatusBadRequest { t.Fatalf("unknown target must be 400 (got %d, body %s)", rr.Code, rr.Body.String()) } } // ── Tier advertisement ─────────────────────────────────────────────────────────────────────── // GET /backup/tiers is the controller's capability probe. Primary must be first and flagged, so a // controller can tell which tier the untargeted endpoints act on. func TestBackupTiers_AdvertisesPrimaryFirst(t *testing.T) { h := tieredServer(t, &fakeStore{}, &fakeBackups{}, &fakeBackups{}).Handler() var resp struct { Data BackupTiersResponse `json:"data"` } rr := do(t, h, "GET", "/backup/tiers", "A", "") if rr.Code != http.StatusOK { t.Fatalf("status = %d", rr.Code) } if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { t.Fatal(err) } if len(resp.Data.Tiers) != 2 { t.Fatalf("want 2 tiers, got %+v", resp.Data.Tiers) } if !resp.Data.Tiers[0].Primary || resp.Data.Tiers[0].Target != "local" { t.Fatalf("primary must be first and flagged; got %+v", resp.Data.Tiers) } if resp.Data.Tiers[1].Target != "felhom-pbs" || resp.Data.Tiers[1].CadenceSeconds != int64((7*24*time.Hour).Seconds()) { t.Fatalf("PBS tier mis-advertised: %+v", resp.Data.Tiers[1]) } } // ── POST /backup routing + per-tier single-flight ──────────────────────────────────────────── // A targeted POST must run THAT tier's runner. Routing both tiers to one runner would silently // write every "PBS" backup to local — a DR tier that reports success and stores nothing. func TestBackupPost_RoutesToTheTargetsOwnRunner(t *testing.T) { local, pbs := &fakeBackups{}, &fakeBackups{} srv := tieredServer(t, &fakeStore{}, local, pbs) h := srv.Handler() if rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted { t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String()) } waitFor(t, func() bool { return len(pbs.seen()) == 1 }) if got := len(local.seen()); got != 0 { t.Fatalf("the LOCAL runner must not have run for a PBS-targeted request (ran %d times)", got) } } // Untargeted POST routes to the primary — the old controller's path. func TestBackupPost_UntargetedRoutesToPrimary(t *testing.T) { local, pbs := &fakeBackups{}, &fakeBackups{} h := tieredServer(t, &fakeStore{}, local, pbs).Handler() if rr := do(t, h, "POST", "/backup", "A", ""); rr.Code != http.StatusAccepted { t.Fatalf("status = %d", rr.Code) } waitFor(t, func() bool { return len(local.seen()) == 1 }) if got := len(pbs.seen()); got != 0 { t.Fatalf("untargeted POST must not touch a non-primary tier (ran %d times)", got) } } // ONE BACKUP AT A TIME PER GUEST (operator ruling 2026-07-26). A second tier's POST while another // tier is still in flight must be REFUSED — vzdump holds the guest lock, so it could not succeed // anyway, and attempting it records a spurious failure that leaves the tier permanently due. // // Crucially it must NOT be handed the busy tier's job id: that is exactly how a caller comes to // believe its own backup ran. func TestBackupPost_SecondTierRefusedWhileAnotherInFlight(t *testing.T) { localGate := make(chan struct{}) local := &fakeBackups{gate: localGate} pbs := &fakeBackups{} h := tieredServer(t, &fakeStore{}, local, pbs).Handler() var first BackupResponse rr1 := do(t, h, "POST", "/backup", "A", "") // local; blocks on the gate if err := json.Unmarshal(rr1.Body.Bytes(), &struct { Data *BackupResponse `json:"data"` }{Data: &first}); err != nil { t.Fatal(err) } waitFor(t, func() bool { return len(local.seen()) == 1 }) rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "") if rr2.Code != http.StatusConflict { t.Fatalf("a second tier must be REFUSED while another is in flight; got %d body %s", rr2.Code, rr2.Body.String()) } // The STRUCTURAL requirement: the refusal must not return a job the caller could mistake for // its own. It is a 409 with ok=false and NO data object, so nothing is parseable as "my job". // (Naming the busy job in the human-readable message is deliberate and useful for diagnosis — // what must never happen is handing it back as BackupResponse.JobID on a 202.) var envelope struct { OK bool `json:"ok"` Data *BackupResponse `json:"data"` Error string `json:"error"` } if err := json.Unmarshal(rr2.Body.Bytes(), &envelope); err != nil { t.Fatal(err) } if envelope.OK { t.Fatalf("a refusal must not be ok=true; body %s", rr2.Body.String()) } if envelope.Data != nil && envelope.Data.JobID != "" { t.Fatalf("the refusal must NOT hand back a job id as the caller's own (got %q); body %s", envelope.Data.JobID, rr2.Body.String()) } if !strings.Contains(rr2.Body.String(), "local") { t.Fatalf("the refusal must NAME the busy tier so the caller can diagnose; body %s", rr2.Body.String()) } if got := len(pbs.seen()); got != 0 { t.Fatalf("the refused tier must NOT have started a backup (ran %d times)", got) } close(localGate) } // Once the busy tier finishes, the other tier may start — and gets its OWN tier-scoped job id. func TestBackupPost_SecondTierAllowedAfterFirstFinishes(t *testing.T) { local, pbs := &fakeBackups{}, &fakeBackups{} h := tieredServer(t, &fakeStore{}, local, pbs).Handler() var first, second BackupResponse if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct { Data *BackupResponse `json:"data"` }{Data: &first}); err != nil { t.Fatal(err) } waitFor(t, func() bool { return len(local.seen()) == 1 }) // Wait for the local job to leave the in-flight phases. waitFor(t, func() bool { rr := do(t, h, "GET", "/backup/status", "A", "") return strings.Contains(rr.Body.String(), `"phase":"done"`) }) rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "") if rr2.Code != http.StatusAccepted { t.Fatalf("after the first tier finished the second must be allowed; got %d body %s", rr2.Code, rr2.Body.String()) } if err := json.Unmarshal(rr2.Body.Bytes(), &struct { Data *BackupResponse `json:"data"` }{Data: &second}); err != nil { t.Fatal(err) } if second.JobID == first.JobID { t.Fatalf("job ids must stay tier-scoped: %q vs %q", first.JobID, second.JobID) } waitFor(t, func() bool { return len(pbs.seen()) == 1 }) } // `snapshotted` still counts as in flight — the vzdump is uploading and still holds the guest lock. // Checking only `running` (the pre-R-82 code) left a window where a second POST started a real // second vzdump. func TestBackupPost_SnapshottedCountsAsInFlight(t *testing.T) { gate := make(chan struct{}) local := &fakeBackups{gate: gate, fireSnapshot: true} // fires onSnapshot, then blocks pbs := &fakeBackups{} h := tieredServer(t, &fakeStore{}, local, pbs).Handler() do(t, h, "POST", "/backup", "A", "") waitFor(t, func() bool { rr := do(t, h, "GET", "/backup/status", "A", "") return strings.Contains(rr.Body.String(), `"phase":"snapshotted"`) }) rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "") if rr.Code != http.StatusConflict { t.Fatalf("a SNAPSHOTTED backup still holds the guest — a second tier must be refused; got %d body %s", rr.Code, rr.Body.String()) } close(gate) } // Same tier, still single-flight: a second POST to a running tier returns the SAME job. func TestBackupPost_SameTierStillSingleFlight(t *testing.T) { gate := make(chan struct{}) local := &fakeBackups{gate: gate} h := tieredServer(t, &fakeStore{}, local, &fakeBackups{}).Handler() var a, b BackupResponse if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct { Data *BackupResponse `json:"data"` }{Data: &a}); err != nil { t.Fatal(err) } if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct { Data *BackupResponse `json:"data"` }{Data: &b}); err != nil { t.Fatal(err) } if a.JobID != b.JobID { t.Fatalf("same tier must single-flight: %q vs %q", a.JobID, b.JobID) } close(gate) } // ── normalizeBackupTiers — the compatibility core ──────────────────────────────────────────── func TestNormalizeBackupTiers(t *testing.T) { svc := &fakeBackups{} t.Run("nil tiers synthesize the legacy single tier", func(t *testing.T) { got := normalizeBackupTiers(nil, svc, 24*time.Hour) if len(got) != 1 || !got[0].Primary || got[0].Cadence != 24*time.Hour { t.Fatalf("legacy synthesis broken: %+v", got) } }) t.Run("primary is hoisted to the front", func(t *testing.T) { got := normalizeBackupTiers([]BackupTier{ {TargetID: "felhom-pbs", Cadence: time.Hour, Service: svc}, {TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc}, }, svc, 24*time.Hour) if len(got) != 2 || got[0].TargetID != "local" || !got[0].Primary || got[1].Primary { t.Fatalf("primary not hoisted / uniqueness broken: %+v", got) } }) t.Run("no primary marked → first becomes primary", func(t *testing.T) { got := normalizeBackupTiers([]BackupTier{ {TargetID: "a", Cadence: time.Hour, Service: svc}, {TargetID: "b", Cadence: time.Hour, Service: svc}, }, svc, 24*time.Hour) if len(got) != 2 || got[0].TargetID != "a" || !got[0].Primary { t.Fatalf("want first-as-primary, got %+v", got) } }) t.Run("a tier with no runner is DROPPED, not advertised", func(t *testing.T) { got := normalizeBackupTiers([]BackupTier{ {TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc}, {TargetID: "felhom-pbs", Cadence: time.Hour, Service: nil}, }, svc, 24*time.Hour) if len(got) != 1 || got[0].TargetID != "local" { t.Fatalf("a serviceless tier must not be advertised (it could never run): %+v", got) } }) } // R-82 Slice D: a tier whose TARGET STORAGE does not exist yet is DEFERRED, not due. // // A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears when // the hub provisions the DR tier. Reporting "due" in that window would have the controller quiesce // the apps and fire a vzdump at a non-existent storage every cadence until provisioning happens. func TestBackupDue_TargetStorageMissing_Defers(t *testing.T) { st := &fakeStore{} // Storage view knows only "local" — the PBS tier's target is not provisioned yet. srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st, Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local", Type: "local"}}}, Tokens: staticTokens{"A": 8200}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}}, }, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatal(err) } srv.baseCtx = context.Background() srv.now = func() time.Time { return testNow } h := srv.Handler() var pbs, local struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if pbs.Data.Due { t.Fatalf("an unprovisioned tier must DEFER, not fire a vzdump at a storage that does not exist; got %+v", pbs.Data) } if !strings.Contains(pbs.Data.Reason, "not present") { t.Fatalf("the deferral must say WHY, or it is indistinguishable from a healthy tier; got %q", pbs.Data.Reason) } // The provisioned tier is unaffected — no evidence yet, so due. if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil { t.Fatal(err) } if !local.Data.Due { t.Fatalf("a PROVISIONED tier with no backup yet must still be due; got %+v", local.Data) } } // A storage-view ERROR must NOT defer. "I could not check" is not "not there" — reading it that way // would silently suppress backups, the absence-is-not-failure rule this project keeps relearning. func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) { srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{}, Storage: errStorage{}, // reused from f2_role_fallback_test.go — Observe always fails Tokens: staticTokens{"A": 8200}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}}, }, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatal(err) } srv.baseCtx = context.Background() srv.now = func() time.Time { return testNow } var pbs struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, srv.Handler(), "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil { t.Fatal(err) } if !pbs.Data.Due { t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data) } } // ── R-84: the cold in-memory store must not cause a redundant backup ───────────────────────── // archiveLister is a fakeBackups that ALSO knows when a backup last landed on its storage. type archiveLister struct { *fakeBackups at time.Time found bool err error } func (a archiveLister) NewestArchiveTime(context.Context, int) (time.Time, bool, error) { return a.at, a.found, a.err } func listerServer(t *testing.T, st *fakeStore, pbsSvc BackupService) http.Handler { t.Helper() srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st, Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}}, Tokens: staticTokens{"A": 8200}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc}, }, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatal(err) } srv.baseCtx = context.Background() srv.now = func() time.Time { return testNow } return srv.Handler() } func dueFor(t *testing.T, h http.Handler, target string) BackupDueResponse { t.Helper() var out struct { Data BackupDueResponse `json:"data"` } if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target="+target, "A", "").Body.Bytes(), &out); err != nil { t.Fatal(err) } return out.Data } // THE R-84 CASE. The in-memory store is EMPTY (the agent just restarted), but the storage holds a // snapshot from 2 hours ago. The tier must NOT be due — otherwise every agent deploy costs a fresh // multi-hour offsite upload. Three redundant local backups were observed on demo-felhom in one // afternoon of deploys before this. // // COMPANION RED-PROOF (observed): delete the newestArchiveOn fold-in from handleBackupDue (the // pre-R-84 shape, in-memory only) → this fails with // "a restart must NOT make the tier due when the storage holds a 2h-old backup; // // got {... Due:true Reason:no successful backup recorded yet ...}". Restored. func TestBackupDue_ColdStore_UsesStorageGroundTruth(t *testing.T) { h := listerServer(t, &fakeStore{}, archiveLister{ fakeBackups: &fakeBackups{}, at: testNow.Add(-2 * time.Hour), found: true, }) got := dueFor(t, h, "felhom-pbs") if got.Due { t.Fatalf("a restart must NOT make the tier due when the storage holds a 2h-old backup; got %+v", got) } if got.AgeSecs == nil || *got.AgeSecs != int64((2*time.Hour).Seconds()) { t.Fatalf("the age must come from the storage; got %+v", got) } } // Ground truth that is genuinely OLD still makes the tier due — this must not become a blanket // suppressor. func TestBackupDue_ColdStore_OldArchiveIsStillDue(t *testing.T) { h := listerServer(t, &fakeStore{}, archiveLister{ fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true, }) if got := dueFor(t, h, "felhom-pbs"); !got.Due { t.Fatalf("a 9-day-old archive under a 7-day cadence MUST still be due; got %+v", got) } } // A storage that genuinely holds nothing → due. The fix must not invent a backup. func TestBackupDue_ColdStore_NoArchiveIsDue(t *testing.T) { h := listerServer(t, &fakeStore{}, archiveLister{fakeBackups: &fakeBackups{}, found: false}) if got := dueFor(t, h, "felhom-pbs"); !got.Due { t.Fatalf("no archive anywhere → due; got %+v", got) } } // A storage-read ERROR must fall back to the in-memory record, NOT be read as "a backup exists". // An unreadable storage must never make a tier look freshly backed up. func TestBackupDue_StorageReadError_DoesNotFakeFreshness(t *testing.T) { h := listerServer(t, &fakeStore{}, archiveLister{ fakeBackups: &fakeBackups{}, at: testNow, found: true, err: errStorageRead, }) if got := dueFor(t, h, "felhom-pbs"); !got.Due { t.Fatalf("a storage-read error must not fake freshness — the in-memory record is empty, so DUE; got %+v", got) } } var errStorageRead = errors.New("simulated storage read failure") // The in-memory record WINS when it is newer than the storage listing — a backup that just finished // this process lifetime is more current than a listing that may lag. func TestBackupDue_InMemoryRecordWinsWhenNewer(t *testing.T) { st := &fakeStore{} st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, true)) // 1h ago, in memory h := listerServer(t, st, archiveLister{ fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true, // stale listing }) got := dueFor(t, h, "felhom-pbs") if got.Due { t.Fatalf("the fresher in-memory record must win over a stale listing; got %+v", got) } } // A service WITHOUT the optional lister degrades to the pre-R-84 behaviour, unchanged. func TestBackupDue_ServiceWithoutLister_UnchangedBehaviour(t *testing.T) { h := listerServer(t, &fakeStore{}, &fakeBackups{}) // plain BackupService if got := dueFor(t, h, "felhom-pbs"); !got.Due || got.Reason != "no successful backup recorded yet" { t.Fatalf("a plain BackupService must behave exactly as before; got %+v", got) } }