felhom-agent/internal/localapi/backup_tiers_test.go

package localapi

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// R-82 Slice A — per-target cadence, due and runner.
//
// The agent and the controller deploy INDEPENDENTLY. A new agent will serve old controllers for as
// long as it takes the fleet to catch up, so the untargeted contract is frozen, not merely
// "probably fine". These tests pin that freeze; the multi-tier behaviour is additive on top.

// tieredServer builds a two-tier server: primary "local" (24h) + "felhom-pbs" (7d), each with its
// own runner, exactly as main.go wires it.
func tieredServer(t *testing.T, st *fakeStore, localSvc, pbsSvc *fakeBackups) *Server {
	t.Helper()
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0",
		Guests:     &fakeGuests{},
		Backups:    localSvc,
		Store:      st,
		// Both tier targets must be PRESENT in the storage view: since v0.102.0 a tier whose target
		// storage is absent DEFERS. A real box has both; a fake with no targets would silently
		// defer every tier and make these assertions vacuous.
		Storage: fakeStorage{targets: []hub.StorageTarget{
			{Name: "local", Type: "local"},
			{Name: "felhom-pbs", Type: "pbs"},
		}},
		Tokens: staticTokens{"A": 8200, "B": 9300},
		BackupTiers: []BackupTier{
			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: localSvc},
			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc},
		},
		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatalf("new server: %v", err)
	}
	srv.baseCtx = context.Background()
	srv.now = func() time.Time { return testNow }
	return srv
}

// seen returns the vmids this fake runner was invoked for (mutex-guarded — the backup runs on a
// goroutine).
func (f *fakeBackups) seen() []int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]int(nil), f.vmids...)
}

// waitFor polls cond for up to 2s. POST /backup is fire-and-forget, so the assertion has to wait
// for the goroutine rather than assume it has run.
func waitFor(t *testing.T, cond func() bool) {
	t.Helper()
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if cond() {
			return
		}
		time.Sleep(5 * time.Millisecond)
	}
	t.Fatalf("condition not met within 2s")
}

func backupAt(target string, vmid int, ago time.Duration, ok bool) hub.Backup {
	return hub.Backup{
		TargetID:  target,
		VMID:      vmid,
		Success:   ok,
		StartedAt: testNow.Add(-ago).Format(time.RFC3339),
	}
}

// ── RED-PROOF 1 — old controller ↔ new agent ────────────────────────────────────────────────
//
// An old controller sends `GET /backup/due` with no query string and parses the pre-R-82 response.
// The response must be BYTE-IDENTICAL — not merely semantically similar. A stray `"target":"local"`
// key is harmless to a tolerant JSON decoder and fatal to a strict one, and we do not get to choose
// which the deployed fleet has.
//
// COMPANION RED-PROOF (observed): drop the `omitempty` from BackupDueResponse.Target and have
// tierFromRequest echo the primary's id for an untargeted request → this test fails with the
// observed body carrying `"target":"local"`. Restored.
func TestBackupDue_Untargeted_ResponseBytesUnchanged(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true))
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	rr := do(t, h, "GET", "/backup/due", "A", "")
	if rr.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rr.Code)
	}
	var got map[string]any
	if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
		t.Fatalf("unmarshal: %v (body %s)", err, rr.Body.String())
	}
	data, _ := got["data"].(map[string]any)
	if data == nil {
		t.Fatalf("no data object in %s", rr.Body.String())
	}
	if _, present := data["target"]; present {
		t.Fatalf("UNTARGETED response MUST NOT carry a target key — an old controller sees a changed contract; body: %s", rr.Body.String())
	}
	if data["due"] != false {
		t.Fatalf("2h-old local backup under a 24h cadence must not be due; body: %s", rr.Body.String())
	}
	if data["reason"] != "within cadence window" {
		t.Fatalf("reason string changed: %v", data["reason"])
	}
}

// The untargeted verdict must come from the PRIMARY tier's cadence, not from whichever tier
// happens to be freshest. With a stale local and a fresh PBS backup, untargeted must say DUE.
func TestBackupDue_Untargeted_UsesPrimaryTierOnly(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("local", 8200, 30*time.Hour, true))     // stale for 24h cadence
	st.RecordBackup(backupAt("felhom-pbs", 8200, 1*time.Hour, true)) // fresh, different tier
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	var resp struct {
		Data BackupDueResponse `json:"data"`
	}
	rr := do(t, h, "GET", "/backup/due", "A", "")
	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
		t.Fatal(err)
	}
	if !resp.Data.Due {
		t.Fatalf("a fresh backup on ANOTHER tier must not satisfy the primary's cadence; got %+v", resp.Data)
	}
}

// ── Per-tier due-ness ────────────────────────────────────────────────────────────────────────

// THE POINT OF THE WHOLE SLICE: a fresh daily local backup must NOT satisfy the weekly PBS tier.
// Without the per-target filter in latestSuccessfulBackupForTarget the DR tier would never run —
// which is exactly today's "applied and empty" state, re-created in code.
func TestBackupDue_PerTier_LocalFreshDoesNotSatisfyPBS(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true))         // fresh daily
	st.RecordBackup(backupAt("felhom-pbs", 8200, 8*24*time.Hour, true)) // 8d — past the 7d weekly
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	var local, pbs struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
		t.Fatal(err)
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if local.Data.Due {
		t.Fatalf("local tier: 2h old under 24h cadence must NOT be due; got %+v", local.Data)
	}
	if !pbs.Data.Due {
		t.Fatalf("PBS tier: 8d old under a 7d cadence MUST be due; got %+v", pbs.Data)
	}
	if pbs.Data.Target != "felhom-pbs" || local.Data.Target != "local" {
		t.Fatalf("a targeted response must echo its tier; got local=%q pbs=%q", local.Data.Target, pbs.Data.Target)
	}
}

// A 6-day-old PBS snapshot is INSIDE the weekly window — it must not be due. The mirror of the
// hub-side threshold test in Slice C, asserted here at the source of truth.
func TestBackupDue_PerTier_PBSWithinWeeklyWindow(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("felhom-pbs", 8200, 6*24*time.Hour, true))
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	var pbs struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if pbs.Data.Due {
		t.Fatalf("6d old under a 7d cadence must NOT be due; got %+v", pbs.Data)
	}
}

// A failed backup must not satisfy any tier's cadence (pre-existing rule, re-asserted per-tier).
func TestBackupDue_PerTier_FailedBackupDoesNotCount(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, false))
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	var pbs struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if !pbs.Data.Due {
		t.Fatalf("a FAILED backup must not satisfy the cadence; got %+v", pbs.Data)
	}
}

// The fail-safe-toward-due rule survives per-tier: an unparseable timestamp yields DUE.
// A spurious backup is cheap; a skipped one is not.
func TestBackupDue_PerTier_UnparseableTimestampIsDue(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(hub.Backup{TargetID: "felhom-pbs", VMID: 8200, Success: true, StartedAt: "not-a-time"})
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	var pbs struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if !pbs.Data.Due {
		t.Fatalf("unparseable timestamp must fail SAFE toward due; got %+v", pbs.Data)
	}
}

// An unknown target is a 400 — never a silent fallback to the primary. A controller asking about a
// tier this agent does not serve must find out, not be handed a different tier's freshness and act
// on it.
func TestBackupDue_UnknownTarget_IsAnErrorNotAFallback(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true))
	h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()

	rr := do(t, h, "GET", "/backup/due?target=does-not-exist", "A", "")
	if rr.Code != http.StatusBadRequest {
		t.Fatalf("unknown target must be 400 (got %d, body %s)", rr.Code, rr.Body.String())
	}
}

// ── Tier advertisement ───────────────────────────────────────────────────────────────────────

// GET /backup/tiers is the controller's capability probe. Primary must be first and flagged, so a
// controller can tell which tier the untargeted endpoints act on.
func TestBackupTiers_AdvertisesPrimaryFirst(t *testing.T) {
	h := tieredServer(t, &fakeStore{}, &fakeBackups{}, &fakeBackups{}).Handler()
	var resp struct {
		Data BackupTiersResponse `json:"data"`
	}
	rr := do(t, h, "GET", "/backup/tiers", "A", "")
	if rr.Code != http.StatusOK {
		t.Fatalf("status = %d", rr.Code)
	}
	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
		t.Fatal(err)
	}
	if len(resp.Data.Tiers) != 2 {
		t.Fatalf("want 2 tiers, got %+v", resp.Data.Tiers)
	}
	if !resp.Data.Tiers[0].Primary || resp.Data.Tiers[0].Target != "local" {
		t.Fatalf("primary must be first and flagged; got %+v", resp.Data.Tiers)
	}
	if resp.Data.Tiers[1].Target != "felhom-pbs" || resp.Data.Tiers[1].CadenceSeconds != int64((7*24*time.Hour).Seconds()) {
		t.Fatalf("PBS tier mis-advertised: %+v", resp.Data.Tiers[1])
	}
}

// ── POST /backup routing + per-tier single-flight ────────────────────────────────────────────

// A targeted POST must run THAT tier's runner. Routing both tiers to one runner would silently
// write every "PBS" backup to local — a DR tier that reports success and stores nothing.
func TestBackupPost_RoutesToTheTargetsOwnRunner(t *testing.T) {
	local, pbs := &fakeBackups{}, &fakeBackups{}
	srv := tieredServer(t, &fakeStore{}, local, pbs)
	h := srv.Handler()

	if rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted {
		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
	}
	waitFor(t, func() bool { return len(pbs.seen()) == 1 })

	if got := len(local.seen()); got != 0 {
		t.Fatalf("the LOCAL runner must not have run for a PBS-targeted request (ran %d times)", got)
	}
}

// Untargeted POST routes to the primary — the old controller's path.
func TestBackupPost_UntargetedRoutesToPrimary(t *testing.T) {
	local, pbs := &fakeBackups{}, &fakeBackups{}
	h := tieredServer(t, &fakeStore{}, local, pbs).Handler()

	if rr := do(t, h, "POST", "/backup", "A", ""); rr.Code != http.StatusAccepted {
		t.Fatalf("status = %d", rr.Code)
	}
	waitFor(t, func() bool { return len(local.seen()) == 1 })
	if got := len(pbs.seen()); got != 0 {
		t.Fatalf("untargeted POST must not touch a non-primary tier (ran %d times)", got)
	}
}

// ONE BACKUP AT A TIME PER GUEST (operator ruling 2026-07-26). A second tier's POST while another
// tier is still in flight must be REFUSED — vzdump holds the guest lock, so it could not succeed
// anyway, and attempting it records a spurious failure that leaves the tier permanently due.
//
// Crucially it must NOT be handed the busy tier's job id: that is exactly how a caller comes to
// believe its own backup ran.
func TestBackupPost_SecondTierRefusedWhileAnotherInFlight(t *testing.T) {
	localGate := make(chan struct{})
	local := &fakeBackups{gate: localGate}
	pbs := &fakeBackups{}
	h := tieredServer(t, &fakeStore{}, local, pbs).Handler()

	var first BackupResponse
	rr1 := do(t, h, "POST", "/backup", "A", "") // local; blocks on the gate
	if err := json.Unmarshal(rr1.Body.Bytes(), &struct {
		Data *BackupResponse `json:"data"`
	}{Data: &first}); err != nil {
		t.Fatal(err)
	}
	waitFor(t, func() bool { return len(local.seen()) == 1 })

	rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
	if rr2.Code != http.StatusConflict {
		t.Fatalf("a second tier must be REFUSED while another is in flight; got %d body %s", rr2.Code, rr2.Body.String())
	}
	// The STRUCTURAL requirement: the refusal must not return a job the caller could mistake for
	// its own. It is a 409 with ok=false and NO data object, so nothing is parseable as "my job".
	// (Naming the busy job in the human-readable message is deliberate and useful for diagnosis —
	// what must never happen is handing it back as BackupResponse.JobID on a 202.)
	var envelope struct {
		OK    bool            `json:"ok"`
		Data  *BackupResponse `json:"data"`
		Error string          `json:"error"`
	}
	if err := json.Unmarshal(rr2.Body.Bytes(), &envelope); err != nil {
		t.Fatal(err)
	}
	if envelope.OK {
		t.Fatalf("a refusal must not be ok=true; body %s", rr2.Body.String())
	}
	if envelope.Data != nil && envelope.Data.JobID != "" {
		t.Fatalf("the refusal must NOT hand back a job id as the caller's own (got %q); body %s",
			envelope.Data.JobID, rr2.Body.String())
	}
	if !strings.Contains(rr2.Body.String(), "local") {
		t.Fatalf("the refusal must NAME the busy tier so the caller can diagnose; body %s", rr2.Body.String())
	}
	if got := len(pbs.seen()); got != 0 {
		t.Fatalf("the refused tier must NOT have started a backup (ran %d times)", got)
	}
	close(localGate)
}

// Once the busy tier finishes, the other tier may start — and gets its OWN tier-scoped job id.
func TestBackupPost_SecondTierAllowedAfterFirstFinishes(t *testing.T) {
	local, pbs := &fakeBackups{}, &fakeBackups{}
	h := tieredServer(t, &fakeStore{}, local, pbs).Handler()

	var first, second BackupResponse
	if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
		Data *BackupResponse `json:"data"`
	}{Data: &first}); err != nil {
		t.Fatal(err)
	}
	waitFor(t, func() bool { return len(local.seen()) == 1 })
	// Wait for the local job to leave the in-flight phases.
	waitFor(t, func() bool {
		rr := do(t, h, "GET", "/backup/status", "A", "")
		return strings.Contains(rr.Body.String(), `"phase":"done"`)
	})

	rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
	if rr2.Code != http.StatusAccepted {
		t.Fatalf("after the first tier finished the second must be allowed; got %d body %s", rr2.Code, rr2.Body.String())
	}
	if err := json.Unmarshal(rr2.Body.Bytes(), &struct {
		Data *BackupResponse `json:"data"`
	}{Data: &second}); err != nil {
		t.Fatal(err)
	}
	if second.JobID == first.JobID {
		t.Fatalf("job ids must stay tier-scoped: %q vs %q", first.JobID, second.JobID)
	}
	waitFor(t, func() bool { return len(pbs.seen()) == 1 })
}

// `snapshotted` still counts as in flight — the vzdump is uploading and still holds the guest lock.
// Checking only `running` (the pre-R-82 code) left a window where a second POST started a real
// second vzdump.
func TestBackupPost_SnapshottedCountsAsInFlight(t *testing.T) {
	gate := make(chan struct{})
	local := &fakeBackups{gate: gate, fireSnapshot: true} // fires onSnapshot, then blocks
	pbs := &fakeBackups{}
	h := tieredServer(t, &fakeStore{}, local, pbs).Handler()

	do(t, h, "POST", "/backup", "A", "")
	waitFor(t, func() bool {
		rr := do(t, h, "GET", "/backup/status", "A", "")
		return strings.Contains(rr.Body.String(), `"phase":"snapshotted"`)
	})

	rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
	if rr.Code != http.StatusConflict {
		t.Fatalf("a SNAPSHOTTED backup still holds the guest — a second tier must be refused; got %d body %s", rr.Code, rr.Body.String())
	}
	close(gate)
}

// Same tier, still single-flight: a second POST to a running tier returns the SAME job.
func TestBackupPost_SameTierStillSingleFlight(t *testing.T) {
	gate := make(chan struct{})
	local := &fakeBackups{gate: gate}
	h := tieredServer(t, &fakeStore{}, local, &fakeBackups{}).Handler()

	var a, b BackupResponse
	if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
		Data *BackupResponse `json:"data"`
	}{Data: &a}); err != nil {
		t.Fatal(err)
	}
	if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
		Data *BackupResponse `json:"data"`
	}{Data: &b}); err != nil {
		t.Fatal(err)
	}
	if a.JobID != b.JobID {
		t.Fatalf("same tier must single-flight: %q vs %q", a.JobID, b.JobID)
	}
	close(gate)
}

// ── normalizeBackupTiers — the compatibility core ────────────────────────────────────────────

func TestNormalizeBackupTiers(t *testing.T) {
	svc := &fakeBackups{}

	t.Run("nil tiers synthesize the legacy single tier", func(t *testing.T) {
		got := normalizeBackupTiers(nil, svc, 24*time.Hour)
		if len(got) != 1 || !got[0].Primary || got[0].Cadence != 24*time.Hour {
			t.Fatalf("legacy synthesis broken: %+v", got)
		}
	})

	t.Run("primary is hoisted to the front", func(t *testing.T) {
		got := normalizeBackupTiers([]BackupTier{
			{TargetID: "felhom-pbs", Cadence: time.Hour, Service: svc},
			{TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc},
		}, svc, 24*time.Hour)
		if len(got) != 2 || got[0].TargetID != "local" || !got[0].Primary || got[1].Primary {
			t.Fatalf("primary not hoisted / uniqueness broken: %+v", got)
		}
	})

	t.Run("no primary marked → first becomes primary", func(t *testing.T) {
		got := normalizeBackupTiers([]BackupTier{
			{TargetID: "a", Cadence: time.Hour, Service: svc},
			{TargetID: "b", Cadence: time.Hour, Service: svc},
		}, svc, 24*time.Hour)
		if len(got) != 2 || got[0].TargetID != "a" || !got[0].Primary {
			t.Fatalf("want first-as-primary, got %+v", got)
		}
	})

	t.Run("a tier with no runner is DROPPED, not advertised", func(t *testing.T) {
		got := normalizeBackupTiers([]BackupTier{
			{TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc},
			{TargetID: "felhom-pbs", Cadence: time.Hour, Service: nil},
		}, svc, 24*time.Hour)
		if len(got) != 1 || got[0].TargetID != "local" {
			t.Fatalf("a serviceless tier must not be advertised (it could never run): %+v", got)
		}
	})
}

// R-82 Slice D: a tier whose TARGET STORAGE does not exist yet is DEFERRED, not due.
//
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears when
// the hub provisions the DR tier. Reporting "due" in that window would have the controller quiesce
// the apps and fire a vzdump at a non-existent storage every cadence until provisioning happens.
func TestBackupDue_TargetStorageMissing_Defers(t *testing.T) {
	st := &fakeStore{}
	// Storage view knows only "local" — the PBS tier's target is not provisioned yet.
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
		Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local", Type: "local"}}},
		Tokens:  staticTokens{"A": 8200},
		BackupTiers: []BackupTier{
			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
		},
		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	srv.now = func() time.Time { return testNow }
	h := srv.Handler()

	var pbs, local struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if pbs.Data.Due {
		t.Fatalf("an unprovisioned tier must DEFER, not fire a vzdump at a storage that does not exist; got %+v", pbs.Data)
	}
	if !strings.Contains(pbs.Data.Reason, "not present") {
		t.Fatalf("the deferral must say WHY, or it is indistinguishable from a healthy tier; got %q", pbs.Data.Reason)
	}
	// The provisioned tier is unaffected — no evidence yet, so due.
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
		t.Fatal(err)
	}
	if !local.Data.Due {
		t.Fatalf("a PROVISIONED tier with no backup yet must still be due; got %+v", local.Data)
	}
}

// A storage-view ERROR must NOT defer. "I could not check" is not "not there" — reading it that way
// would silently suppress backups, the absence-is-not-failure rule this project keeps relearning.
func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) {
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
		Storage: errStorage{}, // reused from f2_role_fallback_test.go — Observe always fails
		Tokens:  staticTokens{"A": 8200},
		BackupTiers: []BackupTier{
			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
		},
		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	srv.now = func() time.Time { return testNow }

	var pbs struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, srv.Handler(), "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
		t.Fatal(err)
	}
	if !pbs.Data.Due {
		t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data)
	}
}

// ── R-84: the cold in-memory store must not cause a redundant backup ─────────────────────────

// archiveLister is a fakeBackups that ALSO knows when a backup last landed on its storage.
type archiveLister struct {
	*fakeBackups
	at    time.Time
	found bool
	err   error
}

func (a archiveLister) NewestArchiveTime(context.Context, int) (time.Time, bool, error) {
	return a.at, a.found, a.err
}

func listerServer(t *testing.T, st *fakeStore, pbsSvc BackupService) http.Handler {
	t.Helper()
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
		Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}},
		Tokens:  staticTokens{"A": 8200},
		BackupTiers: []BackupTier{
			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc},
		},
		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	srv.now = func() time.Time { return testNow }
	return srv.Handler()
}

func dueFor(t *testing.T, h http.Handler, target string) BackupDueResponse {
	t.Helper()
	var out struct {
		Data BackupDueResponse `json:"data"`
	}
	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target="+target, "A", "").Body.Bytes(), &out); err != nil {
		t.Fatal(err)
	}
	return out.Data
}

// THE R-84 CASE. The in-memory store is EMPTY (the agent just restarted), but the storage holds a
// snapshot from 2 hours ago. The tier must NOT be due — otherwise every agent deploy costs a fresh
// multi-hour offsite upload. Three redundant local backups were observed on demo-felhom in one
// afternoon of deploys before this.
//
// COMPANION RED-PROOF (observed): delete the newestArchiveOn fold-in from handleBackupDue (the
// pre-R-84 shape, in-memory only) → this fails with
// "a restart must NOT make the tier due when the storage holds a 2h-old backup;
//
//	got {... Due:true Reason:no successful backup recorded yet ...}". Restored.
func TestBackupDue_ColdStore_UsesStorageGroundTruth(t *testing.T) {
	h := listerServer(t, &fakeStore{}, archiveLister{
		fakeBackups: &fakeBackups{}, at: testNow.Add(-2 * time.Hour), found: true,
	})
	got := dueFor(t, h, "felhom-pbs")
	if got.Due {
		t.Fatalf("a restart must NOT make the tier due when the storage holds a 2h-old backup; got %+v", got)
	}
	if got.AgeSecs == nil || *got.AgeSecs != int64((2*time.Hour).Seconds()) {
		t.Fatalf("the age must come from the storage; got %+v", got)
	}
}

// Ground truth that is genuinely OLD still makes the tier due — this must not become a blanket
// suppressor.
func TestBackupDue_ColdStore_OldArchiveIsStillDue(t *testing.T) {
	h := listerServer(t, &fakeStore{}, archiveLister{
		fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true,
	})
	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
		t.Fatalf("a 9-day-old archive under a 7-day cadence MUST still be due; got %+v", got)
	}
}

// A storage that genuinely holds nothing → due. The fix must not invent a backup.
func TestBackupDue_ColdStore_NoArchiveIsDue(t *testing.T) {
	h := listerServer(t, &fakeStore{}, archiveLister{fakeBackups: &fakeBackups{}, found: false})
	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
		t.Fatalf("no archive anywhere → due; got %+v", got)
	}
}

// A storage-read ERROR must fall back to the in-memory record, NOT be read as "a backup exists".
// An unreadable storage must never make a tier look freshly backed up.
func TestBackupDue_StorageReadError_DoesNotFakeFreshness(t *testing.T) {
	h := listerServer(t, &fakeStore{}, archiveLister{
		fakeBackups: &fakeBackups{}, at: testNow, found: true, err: errStorageRead,
	})
	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
		t.Fatalf("a storage-read error must not fake freshness — the in-memory record is empty, so DUE; got %+v", got)
	}
}

var errStorageRead = errors.New("simulated storage read failure")

// The in-memory record WINS when it is newer than the storage listing — a backup that just finished
// this process lifetime is more current than a listing that may lag.
func TestBackupDue_InMemoryRecordWinsWhenNewer(t *testing.T) {
	st := &fakeStore{}
	st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, true)) // 1h ago, in memory
	h := listerServer(t, st, archiveLister{
		fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true, // stale listing
	})
	got := dueFor(t, h, "felhom-pbs")
	if got.Due {
		t.Fatalf("the fresher in-memory record must win over a stale listing; got %+v", got)
	}
}

// A service WITHOUT the optional lister degrades to the pre-R-84 behaviour, unchanged.
func TestBackupDue_ServiceWithoutLister_UnchangedBehaviour(t *testing.T) {
	h := listerServer(t, &fakeStore{}, &fakeBackups{}) // plain BackupService
	if got := dueFor(t, h, "felhom-pbs"); !got.Due || got.Reason != "no successful backup recorded yet" {
		t.Fatalf("a plain BackupService must behave exactly as before; got %+v", got)
	}
}