felhom-agent/internal/hub/restoretest_merge_test.go

package hub

import (
	"context"
	"testing"
	"time"
)

// R-189 — a passing restore-test must survive an agent restart and reach the hub.
//
// THE OBSERVATION THIS EXISTS FOR (2026-08-03, demo-felhom): a real 14.5 GB offsite restore-test
// PASSED at 15:25:14; the agent was restarted 2 m 43 s later for a deploy; the hub logged
// `0 restore-tests` on the next two host-reports. The in-memory store's own comment said "lost on
// restart; the cadence re-populates", which was true under a timer and stopped being true when R-86
// made the agent refuse to re-test an archive it has already proven.
//
// Timestamps here carry JITTER (odd minutes and seconds, not round hours) — yesterday a test was
// hollow because a perfectly regular series landed exactly on a threshold and passed under the
// mutation it was meant to catch.

type fakeLatest struct{ tests []RestoreTest }

func (f *fakeLatest) RestoreTests(context.Context) []RestoreTest { return f.tests }

type fakeProven struct{ tests []RestoreTest }

func (f *fakeProven) ProvenRestoreTests(context.Context) []RestoreTest { return f.tests }

func rt(tier, archive string, pass bool, at time.Time) RestoreTest {
	return RestoreTest{
		SourceArchive: archive, SourceTier: tier, Pass: pass,
		Verified: "boot+running", TestedAt: at.UTC().Format(time.RFC3339),
	}
}

// mergeCollector builds a Collector with only the two restore-test seams wired — the merge is what
// is under test, not the rest of the collection.
func mergeCollector(latest, proven []RestoreTest) *Collector {
	c := &Collector{}
	if latest != nil {
		c.restoreTests = &fakeLatest{tests: latest}
	}
	if proven != nil {
		c.provenTests = &fakeProven{tests: proven}
	}
	return c
}

func findTier(got []RestoreTest, tier string) (RestoreTest, int) {
	var hit RestoreTest
	n := 0
	for _, e := range got {
		if e.SourceTier == tier {
			hit, n = e, n+1
		}
	}
	return hit, n
}

// ── SCENARIO A — a proof survives a restart and reaches the hub ──────────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-03): delete the `c.provenTests` merge from
// collectRestoreTests (return the in-memory slice as it used to) →
//
//	--- FAIL: TestMerge_ProofSurvivesARestart
//	    restoretest_merge_test.go: after a restart the persisted proof must be reported; got 0 entr(ies)
//
// which is exactly the live observation: `0 restore-tests`. Restored.
func TestMerge_ProofSurvivesARestart(t *testing.T) {
	provenAt := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) // the real run's timestamp
	// After a restart the in-memory store is EMPTY — this is the whole point.
	c := mergeCollector([]RestoreTest{}, []RestoreTest{
		rt("pbs", "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z", true, provenAt),
	})

	got := c.collectRestoreTests(context.Background())
	if len(got) != 1 {
		t.Fatalf("after a restart the persisted proof must be reported; got %d entr(ies): %+v", len(got), got)
	}
	e := got[0]
	if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" {
		t.Fatalf("the entry must name the archive that was proven — the hub keys on it; got %q", e.SourceArchive)
	}
	if e.SourceTier != "pbs" || !e.Pass {
		t.Fatalf("the entry must be a PASS on the tier it was proven on; got tier=%q pass=%v", e.SourceTier, e.Pass)
	}
	if e.TestedAt != provenAt.Format(time.RFC3339) {
		t.Fatalf("the entry must carry the ORIGINAL test time, not now(); got %q", e.TestedAt)
	}
}

// ── SCENARIO B — the report does not invent a pass ───────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): make the state layer emit an entry for an unproven tier (drop the
// `reportable()` filter in ProvenRestoreTests, so a legacy record with no archive is emitted) — the
// equivalent at this layer is a proven-source that returns an entry for a tier nothing proved, which
// this test injects directly and the assertion below rejects.
func TestMerge_NeverInventsAPassForAnUnprovenTier(t *testing.T) {
	// Nothing proven anywhere: no in-memory result, no persisted proof.
	c := mergeCollector([]RestoreTest{}, []RestoreTest{})
	if got := c.collectRestoreTests(context.Background()); len(got) != 0 {
		t.Fatalf("a tier with no proof must produce NO entry — an unproven tier reading as proven is "+
			"worse than the defect being fixed; got %+v", got)
	}

	// And an entry the state layer could not describe (no tier) is never promoted into a proof.
	c2 := mergeCollector([]RestoreTest{}, []RestoreTest{
		{SourceArchive: "local:backup/x.tar.zst", SourceTier: "", Pass: true,
			TestedAt: time.Date(2026, 8, 1, 4, 41, 58, 0, time.UTC).Format(time.RFC3339)},
	})
	if got := c2.collectRestoreTests(context.Background()); len(got) != 0 {
		t.Fatalf("a persisted record with no tier is not a usable proof and must be dropped; got %+v", got)
	}
}

// ── SCENARIO C — a fresh in-memory result wins, and never duplicates ─────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-03): remove the de-duplication (append every persisted entry
// unconditionally) →
//
//	--- FAIL: TestMerge_NewerWinsAndNeverDuplicatesATier
//	    restoretest_merge_test.go: one entry per tier; got 2 for "pbs" — the hub would read two tests
//
// Restored.
func TestMerge_NewerWinsAndNeverDuplicatesATier(t *testing.T) {
	lastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) // jittered, from the real box
	fiveMinAgo := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC)

	c := mergeCollector(
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
	)
	got := c.collectRestoreTests(context.Background())
	e, n := findTier(got, "pbs")
	if n != 1 {
		t.Fatalf("one entry per tier; got %d for \"pbs\" — the hub would read two tests: %+v", n, got)
	}
	if e.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
		t.Fatalf("the NEWER result must win; got %q tested %q", e.SourceArchive, e.TestedAt)
	}

	// ...and the older-in-memory / newer-persisted direction, which is the post-restart case.
	c2 := mergeCollector(
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
	)
	e2, n2 := findTier(c2.collectRestoreTests(context.Background()), "pbs")
	if n2 != 1 || e2.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
		t.Fatalf("newest must win regardless of which source it came from; got %d entr(ies), archive %q", n2, e2.SourceArchive)
	}
}

// ── SCENARIO D — a failure still reaches the hub ─────────────────────────────────────────────
//
// The merge must not mask a failure with an older stored success. A failing tier is retried at the
// next evaluation and its record lives ONLY in memory, so losing it here would silence the loudest
// DR signal this system produces.
func TestMerge_AFailureIsStillReported(t *testing.T) {
	provenLastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC)
	failedJustNow := time.Date(2026, 8, 3, 13, 41, 7, 0, time.UTC)

	c := mergeCollector(
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", false, failedJustNow)},
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, provenLastWeek)},
	)
	e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
	if n != 1 {
		t.Fatalf("one entry per tier; got %d: %+v", n, c.collectRestoreTests(context.Background()))
	}
	if e.Pass {
		t.Fatalf("a FAILURE newer than the stored proof must be what is reported — masking it would "+
			"silence the loudest DR signal there is; got pass=%v archive=%q", e.Pass, e.SourceArchive)
	}
}

// Two different tiers are both reported — the merge is per tier, not a single slot.
func TestMerge_BothTiersSurvive(t *testing.T) {
	c := mergeCollector(
		[]RestoreTest{rt("local", "felhom-backup:backup/vzdump-lxc-9201-a.tar.zst", true,
			time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))},
		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/x", true,
			time.Date(2026, 8, 2, 5, 12, 33, 0, time.UTC))},
	)
	got := c.collectRestoreTests(context.Background())
	if _, n := findTier(got, "local"); n != 1 {
		t.Fatalf("the in-memory tier must survive the merge; got %+v", got)
	}
	if _, n := findTier(got, "pbs"); n != 1 {
		t.Fatalf("the persisted tier must survive the merge; got %+v", got)
	}
}

// A malformed timestamp must never displace a good entry — "unparseable" is not "newest".
func TestMerge_MalformedTimestampNeverWins(t *testing.T) {
	good := rt("pbs", "felhom-pbs:backup/ct/9201/good", true, time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC))
	bad := RestoreTest{SourceArchive: "felhom-pbs:backup/ct/9201/bad", SourceTier: "pbs", Pass: true, TestedAt: "not-a-time"}

	c := mergeCollector([]RestoreTest{good}, []RestoreTest{bad})
	e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
	if n != 1 || e.SourceArchive != "felhom-pbs:backup/ct/9201/good" {
		t.Fatalf("an unparseable timestamp must not displace a good entry; got %d entr(ies), archive %q", n, e.SourceArchive)
	}
}

// A nil proven-source leaves the pre-R-189 behaviour exactly as it was.
func TestMerge_NilProvenSourceIsANoOp(t *testing.T) {
	only := rt("local", "felhom-backup:backup/x.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))
	c := mergeCollector([]RestoreTest{only}, nil)
	got := c.collectRestoreTests(context.Background())
	if len(got) != 1 || got[0].SourceArchive != only.SourceArchive {
		t.Fatalf("a nil durable source must not change anything; got %+v", got)
	}
}