v0.103.0 — R-84: an agent restart no longer triggers a redundant backup

Observed live: three redundant local backups on demo-felhom in one afternoon of
deploys. The backup Store is in-memory ('lost on restart; the cadence
re-populates'), so after every restart /backup/due said 'no successful backup
recorded yet' and the controller took another one. On the offsite tier that is a
wasted multi-hour WAN upload after every agent deploy.

- BackupRunner.NewestArchiveTime: when a backup last LANDED on this tier's
  storage, read from the storage.
- localapi.BackupArchiveLister (optional BackupService extension): the due-check
  takes whichever is newer, the in-memory record or the storage.

Asking the storage rather than persisting the store is deliberate: it is ground
truth (a pruned archive correctly stops counting, where a persisted record would
keep claiming a backup that no longer exists), needs no new on-disk state, and
answers only 'when did a backup last land' — the richer fields stay with real
records so the host-report never carries invented numbers.

Fail-safes: read error -> fall back to memory (never fake freshness, never
suppress); genuinely empty -> due; old archive -> still due; service without the
lister -> unchanged.

Red-proof observed; full suite green (29 packages).

internal/localapi/backup_tiers_test.go

@@ -3,6 +3,7 @@ package localapi
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
 	"log/slog"
 	"net/http"
@@ -546,3 +547,124 @@ func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) {
 		t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data)
 	}
 }
+
+// ── R-84: the cold in-memory store must not cause a redundant backup ─────────────────────────
+
+// archiveLister is a fakeBackups that ALSO knows when a backup last landed on its storage.
+type archiveLister struct {
+	*fakeBackups
+	at    time.Time
+	found bool
+	err   error
+}
+
+func (a archiveLister) NewestArchiveTime(context.Context, int) (time.Time, bool, error) {
+	return a.at, a.found, a.err
+}
+
+func listerServer(t *testing.T, st *fakeStore, pbsSvc BackupService) http.Handler {
+	t.Helper()
+	srv, err := NewServer(Options{
+		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
+		Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}},
+		Tokens:  staticTokens{"A": 8200},
+		BackupTiers: []BackupTier{
+			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
+			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc},
+		},
+		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv.baseCtx = context.Background()
+	srv.now = func() time.Time { return testNow }
+	return srv.Handler()
+}
+
+func dueFor(t *testing.T, h http.Handler, target string) BackupDueResponse {
+	t.Helper()
+	var out struct {
+		Data BackupDueResponse `json:"data"`
+	}
+	if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target="+target, "A", "").Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	return out.Data
+}
+
+// THE R-84 CASE. The in-memory store is EMPTY (the agent just restarted), but the storage holds a
+// snapshot from 2 hours ago. The tier must NOT be due — otherwise every agent deploy costs a fresh
+// multi-hour offsite upload. Three redundant local backups were observed on demo-felhom in one
+// afternoon of deploys before this.
+//
+// COMPANION RED-PROOF (observed): delete the newestArchiveOn fold-in from handleBackupDue (the
+// pre-R-84 shape, in-memory only) → this fails with
+// "a restart must NOT make the tier due when the storage holds a 2h-old backup;
+//  got {... Due:true Reason:no successful backup recorded yet ...}". Restored.
+func TestBackupDue_ColdStore_UsesStorageGroundTruth(t *testing.T) {
+	h := listerServer(t, &fakeStore{}, archiveLister{
+		fakeBackups: &fakeBackups{}, at: testNow.Add(-2 * time.Hour), found: true,
+	})
+	got := dueFor(t, h, "felhom-pbs")
+	if got.Due {
+		t.Fatalf("a restart must NOT make the tier due when the storage holds a 2h-old backup; got %+v", got)
+	}
+	if got.AgeSecs == nil || *got.AgeSecs != int64((2 * time.Hour).Seconds()) {
+		t.Fatalf("the age must come from the storage; got %+v", got)
+	}
+}
+
+// Ground truth that is genuinely OLD still makes the tier due — this must not become a blanket
+// suppressor.
+func TestBackupDue_ColdStore_OldArchiveIsStillDue(t *testing.T) {
+	h := listerServer(t, &fakeStore{}, archiveLister{
+		fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true,
+	})
+	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
+		t.Fatalf("a 9-day-old archive under a 7-day cadence MUST still be due; got %+v", got)
+	}
+}
+
+// A storage that genuinely holds nothing → due. The fix must not invent a backup.
+func TestBackupDue_ColdStore_NoArchiveIsDue(t *testing.T) {
+	h := listerServer(t, &fakeStore{}, archiveLister{fakeBackups: &fakeBackups{}, found: false})
+	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
+		t.Fatalf("no archive anywhere → due; got %+v", got)
+	}
+}
+
+// A storage-read ERROR must fall back to the in-memory record, NOT be read as "a backup exists".
+// An unreadable storage must never make a tier look freshly backed up.
+func TestBackupDue_StorageReadError_DoesNotFakeFreshness(t *testing.T) {
+	h := listerServer(t, &fakeStore{}, archiveLister{
+		fakeBackups: &fakeBackups{}, at: testNow, found: true, err: errStorageRead,
+	})
+	if got := dueFor(t, h, "felhom-pbs"); !got.Due {
+		t.Fatalf("a storage-read error must not fake freshness — the in-memory record is empty, so DUE; got %+v", got)
+	}
+}
+
+var errStorageRead = errors.New("simulated storage read failure")
+
+// The in-memory record WINS when it is newer than the storage listing — a backup that just finished
+// this process lifetime is more current than a listing that may lag.
+func TestBackupDue_InMemoryRecordWinsWhenNewer(t *testing.T) {
+	st := &fakeStore{}
+	st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, true)) // 1h ago, in memory
+	h := listerServer(t, st, archiveLister{
+		fakeBackups: &fakeBackups{}, at: testNow.Add(-9 * 24 * time.Hour), found: true, // stale listing
+	})
+	got := dueFor(t, h, "felhom-pbs")
+	if got.Due {
+		t.Fatalf("the fresher in-memory record must win over a stale listing; got %+v", got)
+	}
+}
+
+// A service WITHOUT the optional lister degrades to the pre-R-84 behaviour, unchanged.
+func TestBackupDue_ServiceWithoutLister_UnchangedBehaviour(t *testing.T) {
+	h := listerServer(t, &fakeStore{}, &fakeBackups{}) // plain BackupService
+	if got := dueFor(t, h, "felhom-pbs"); !got.Due || got.Reason != "no successful backup recorded yet" {
+		t.Fatalf("a plain BackupService must behave exactly as before; got %+v", got)
+	}
+}