v0.122.0: three ways the signals lied about themselves (R-189, R-188, R-186)

All three are the reporting and release path misreporting its own work. No customer machine, no backup, no restore, no data. The restore-test itself and when it runs are unchanged. R-189 — a passing restore-test no longer vanishes on a restart. restore_tests[] came only from the in-memory store, whose comment ("lost on restart; the cadence re-populates") was true under a timer and stopped being true when R-86 made the agent refuse to re-test a proven archive: the proof is then not repeated for a whole archive generation. Observed live — a 14.5 GB offsite PASS reached no host-report because the agent was restarted 2m43s later. RestoreTestState now carries tier + verified beside the archive and renders reportable entries; the collector merges them, one per tier, newest by TestedAt. It refuses to lie: a record missing archive-or-tier produces no entry, and run mechanics are not re-invented. Only successes are persisted, and the asymmetry is now written where it will be read. R-188 — a correct release stops emailing a failure. Only the tag PUSH moved (build -> tag locally -> publish -> push tag): the push wakes CI, and a tag visible before its package made the gate correctly fail a correct release about half the time. The old order's invariant is asserted directly instead — the gate now refuses a published version with no tag, as a bounded probe that prints its own coverage, because the package listing api is still 401 without a token. R-186 — a released binary can be verified by rebuilding it. -trimpath -buildvcs=false: same source, same bytes, tag or no tag. Measured. publish-agent's fallback also forced CGO_ENABLED=0 and produced a 74 KB different binary for the same version; both paths now build identically. CLAUDE.md records the command.
2026-08-03 16:40:18 +02:00
parent 3d0a1d615d
commit 7581f8140a
16 changed files with 895 additions and 42 deletions
@@ -47,6 +47,22 @@ type RestoreTestReporter interface {
 	RestoreTests(ctx context.Context) []RestoreTest
 }

+// ProvenRestoreTestReporter is the DURABLE half of the restore-test signal (R-189).
+//
+// RestoreTestReporter above is backed by an in-memory store whose own comment used to read "lost on
+// restart; the cadence re-populates". That was true while a timer re-tested every tier daily. It
+// stopped being true on 2026-08-03: under per-archive due-ness the agent will not re-test an archive
+// it has already proven, so a proof lost to a restart is not repeated for a whole archive generation
+// — a week on the offsite tier — and the hub reports the tier unproven the entire time.
+//
+// Observed, not predicted: a real 14.5 GB offsite restore passed at 15:25:14, the agent was restarted
+// 2 m 43 s later for a deploy, and the hub logged `0 restore-tests` on the next two reports.
+//
+// (*backup.RestoreTestState).ProvenRestoreTests satisfies this. nil → the merge is a no-op.
+type ProvenRestoreTestReporter interface {
+	ProvenRestoreTests(ctx context.Context) []RestoreTest
+}
+
 // PBSReporter is the slice-6-Phase-B seam the pbs verify loop plugs into (same pattern).
 // Returns the agent's latest-known PBS snapshot inventory + verify-state. nil → empty.
 type PBSReporter interface {
@@ -79,6 +95,7 @@ type Collector struct {
 	storage      StorageObserver
 	backups      BackupReporter
 	restoreTests RestoreTestReporter
+	provenTests  ProvenRestoreTestReporter
 	pbs          PBSReporter
 	temp         TempReader                                    // slice 9: host CPU/chassis temp (nil-safe → nil temp)
 	capProbe     func(ctx context.Context) []capability.Status // v0.44.0: privileged-capability self-check (nil → empty)
@@ -427,16 +444,90 @@ func (c *Collector) collectBackups(ctx context.Context) []Backup {
 	return []Backup{}
 }

+// collectRestoreTests merges the in-memory result with the PERSISTED per-tier proofs (R-189).
+//
+// The rule is ONE ENTRY PER TIER, NEWEST WINS, and it falls out of what each source means rather
+// than from a preference between them:
+//
+//   - the in-memory store holds this process's latest run, pass OR fail. A failure exists nowhere
+//     else and must always reach the hub — a failing tier is retried at the next evaluation, so its
+//     record is short-lived by design;
+//   - the persisted state holds the last SUCCESS per tier and survives a restart.
+//
+// Comparing by TestedAt gives the right answer in every case without special-casing: a fresh failure
+// beats an older stored success (the failure is the news), a stored success beats a stale in-memory
+// entry after a restart, and a tier proved twice never appears twice — two entries for one tier would
+// read at the hub as two tests.
+//
+// A tier with no usable persisted proof contributes NOTHING. Reporting an unproven tier as proven
+// would be a worse defect than the one this closes.
 func (c *Collector) collectRestoreTests(ctx context.Context) []RestoreTest {
-	if c.restoreTests == nil {
-		return []RestoreTest{}
+	out := []RestoreTest{}
+	if c.restoreTests != nil {
+		if r := c.restoreTests.RestoreTests(ctx); r != nil {
+			out = append(out, r...)
+		}
 	}
-	if r := c.restoreTests.RestoreTests(ctx); r != nil {
-		return r
+	if c.provenTests == nil {
+		return out
 	}
-	return []RestoreTest{}
+
+	// Index what we already have by tier, keeping the newest per tier.
+	best := map[string]int{} // tier → index into out
+	for i, rt := range out {
+		if rt.SourceTier == "" {
+			continue // untiered entry: never deduped, never overwritten — we cannot say what it is
+		}
+		if j, seen := best[rt.SourceTier]; !seen || newerRestoreTest(rt, out[j]) {
+			best[rt.SourceTier] = i
+		}
+	}
+	for _, p := range c.provenTests.ProvenRestoreTests(ctx) {
+		if p.SourceTier == "" {
+			continue // not usable as a per-tier proof; the state layer already filters these
+		}
+		i, seen := best[p.SourceTier]
+		if !seen {
+			out = append(out, p)
+			best[p.SourceTier] = len(out) - 1
+			continue
+		}
+		if newerRestoreTest(p, out[i]) {
+			out[i] = p
+		}
+	}
+	return out
 }

+// newerRestoreTest reports whether a was tested after b. An unparseable or absent timestamp is
+// treated as OLDER, so a malformed entry can never displace a good one.
+func newerRestoreTest(a, b RestoreTest) bool {
+	ta, aok := parseRestoreTestedAt(a.TestedAt)
+	tb, bok := parseRestoreTestedAt(b.TestedAt)
+	if !aok {
+		return false
+	}
+	if !bok {
+		return true
+	}
+	return ta.After(tb)
+}
+
+func parseRestoreTestedAt(s string) (time.Time, bool) {
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return t.UTC(), true
+}
+
+// SetProvenRestoreTests wires the durable proof source. It is a setter rather than a constructor
+// argument because the persisted state is opened later in main() than the collector is built; the
+// same shape as the other late-wired seams here. **The wiring is asserted by an AST test** — the
+// method it feeds carried a doc comment naming a "host-report gauge" for weeks with no caller at
+// all, and this fix must not become the next instance of that.
+func (c *Collector) SetProvenRestoreTests(p ProvenRestoreTestReporter) { c.provenTests = p }
+
 // collectPBSSnapshots reads the latest PBS snapshot inventory via the seam (nil → empty).
 func (c *Collector) collectPBSSnapshots(ctx context.Context) []PBSSnapshot {
 	if c.pbs == nil {
@@ -0,0 +1,213 @@
+package hub
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// R-189 — a passing restore-test must survive an agent restart and reach the hub.
+//
+// THE OBSERVATION THIS EXISTS FOR (2026-08-03, demo-felhom): a real 14.5 GB offsite restore-test
+// PASSED at 15:25:14; the agent was restarted 2 m 43 s later for a deploy; the hub logged
+// `0 restore-tests` on the next two host-reports. The in-memory store's own comment said "lost on
+// restart; the cadence re-populates", which was true under a timer and stopped being true when R-86
+// made the agent refuse to re-test an archive it has already proven.
+//
+// Timestamps here carry JITTER (odd minutes and seconds, not round hours) — yesterday a test was
+// hollow because a perfectly regular series landed exactly on a threshold and passed under the
+// mutation it was meant to catch.
+
+type fakeLatest struct{ tests []RestoreTest }
+
+func (f *fakeLatest) RestoreTests(context.Context) []RestoreTest { return f.tests }
+
+type fakeProven struct{ tests []RestoreTest }
+
+func (f *fakeProven) ProvenRestoreTests(context.Context) []RestoreTest { return f.tests }
+
+func rt(tier, archive string, pass bool, at time.Time) RestoreTest {
+	return RestoreTest{
+		SourceArchive: archive, SourceTier: tier, Pass: pass,
+		Verified: "boot+running", TestedAt: at.UTC().Format(time.RFC3339),
+	}
+}
+
+// mergeCollector builds a Collector with only the two restore-test seams wired — the merge is what
+// is under test, not the rest of the collection.
+func mergeCollector(latest, proven []RestoreTest) *Collector {
+	c := &Collector{}
+	if latest != nil {
+		c.restoreTests = &fakeLatest{tests: latest}
+	}
+	if proven != nil {
+		c.provenTests = &fakeProven{tests: proven}
+	}
+	return c
+}
+
+func findTier(got []RestoreTest, tier string) (RestoreTest, int) {
+	var hit RestoreTest
+	n := 0
+	for _, e := range got {
+		if e.SourceTier == tier {
+			hit, n = e, n+1
+		}
+	}
+	return hit, n
+}
+
+// ── SCENARIO A — a proof survives a restart and reaches the hub ──────────────────────────────
+//
+// COMPANION RED-PROOF (observed 2026-08-03): delete the `c.provenTests` merge from
+// collectRestoreTests (return the in-memory slice as it used to) →
+//
+//	--- FAIL: TestMerge_ProofSurvivesARestart
+//	    restoretest_merge_test.go: after a restart the persisted proof must be reported; got 0 entr(ies)
+//
+// which is exactly the live observation: `0 restore-tests`. Restored.
+func TestMerge_ProofSurvivesARestart(t *testing.T) {
+	provenAt := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) // the real run's timestamp
+	// After a restart the in-memory store is EMPTY — this is the whole point.
+	c := mergeCollector([]RestoreTest{}, []RestoreTest{
+		rt("pbs", "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z", true, provenAt),
+	})
+
+	got := c.collectRestoreTests(context.Background())
+	if len(got) != 1 {
+		t.Fatalf("after a restart the persisted proof must be reported; got %d entr(ies): %+v", len(got), got)
+	}
+	e := got[0]
+	if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" {
+		t.Fatalf("the entry must name the archive that was proven — the hub keys on it; got %q", e.SourceArchive)
+	}
+	if e.SourceTier != "pbs" || !e.Pass {
+		t.Fatalf("the entry must be a PASS on the tier it was proven on; got tier=%q pass=%v", e.SourceTier, e.Pass)
+	}
+	if e.TestedAt != provenAt.Format(time.RFC3339) {
+		t.Fatalf("the entry must carry the ORIGINAL test time, not now(); got %q", e.TestedAt)
+	}
+}
+
+// ── SCENARIO B — the report does not invent a pass ───────────────────────────────────────────
+//
+// COMPANION RED-PROOF (observed): make the state layer emit an entry for an unproven tier (drop the
+// `reportable()` filter in ProvenRestoreTests, so a legacy record with no archive is emitted) — the
+// equivalent at this layer is a proven-source that returns an entry for a tier nothing proved, which
+// this test injects directly and the assertion below rejects.
+func TestMerge_NeverInventsAPassForAnUnprovenTier(t *testing.T) {
+	// Nothing proven anywhere: no in-memory result, no persisted proof.
+	c := mergeCollector([]RestoreTest{}, []RestoreTest{})
+	if got := c.collectRestoreTests(context.Background()); len(got) != 0 {
+		t.Fatalf("a tier with no proof must produce NO entry — an unproven tier reading as proven is "+
+			"worse than the defect being fixed; got %+v", got)
+	}
+
+	// And an entry the state layer could not describe (no tier) is never promoted into a proof.
+	c2 := mergeCollector([]RestoreTest{}, []RestoreTest{
+		{SourceArchive: "local:backup/x.tar.zst", SourceTier: "", Pass: true,
+			TestedAt: time.Date(2026, 8, 1, 4, 41, 58, 0, time.UTC).Format(time.RFC3339)},
+	})
+	if got := c2.collectRestoreTests(context.Background()); len(got) != 0 {
+		t.Fatalf("a persisted record with no tier is not a usable proof and must be dropped; got %+v", got)
+	}
+}
+
+// ── SCENARIO C — a fresh in-memory result wins, and never duplicates ─────────────────────────
+//
+// COMPANION RED-PROOF (observed 2026-08-03): remove the de-duplication (append every persisted entry
+// unconditionally) →
+//
+//	--- FAIL: TestMerge_NewerWinsAndNeverDuplicatesATier
+//	    restoretest_merge_test.go: one entry per tier; got 2 for "pbs" — the hub would read two tests
+//
+// Restored.
+func TestMerge_NewerWinsAndNeverDuplicatesATier(t *testing.T) {
+	lastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) // jittered, from the real box
+	fiveMinAgo := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC)
+
+	c := mergeCollector(
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
+	)
+	got := c.collectRestoreTests(context.Background())
+	e, n := findTier(got, "pbs")
+	if n != 1 {
+		t.Fatalf("one entry per tier; got %d for \"pbs\" — the hub would read two tests: %+v", n, got)
+	}
+	if e.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
+		t.Fatalf("the NEWER result must win; got %q tested %q", e.SourceArchive, e.TestedAt)
+	}
+
+	// ...and the older-in-memory / newer-persisted direction, which is the post-restart case.
+	c2 := mergeCollector(
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
+	)
+	e2, n2 := findTier(c2.collectRestoreTests(context.Background()), "pbs")
+	if n2 != 1 || e2.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
+		t.Fatalf("newest must win regardless of which source it came from; got %d entr(ies), archive %q", n2, e2.SourceArchive)
+	}
+}
+
+// ── SCENARIO D — a failure still reaches the hub ─────────────────────────────────────────────
+//
+// The merge must not mask a failure with an older stored success. A failing tier is retried at the
+// next evaluation and its record lives ONLY in memory, so losing it here would silence the loudest
+// DR signal this system produces.
+func TestMerge_AFailureIsStillReported(t *testing.T) {
+	provenLastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC)
+	failedJustNow := time.Date(2026, 8, 3, 13, 41, 7, 0, time.UTC)
+
+	c := mergeCollector(
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", false, failedJustNow)},
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, provenLastWeek)},
+	)
+	e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
+	if n != 1 {
+		t.Fatalf("one entry per tier; got %d: %+v", n, c.collectRestoreTests(context.Background()))
+	}
+	if e.Pass {
+		t.Fatalf("a FAILURE newer than the stored proof must be what is reported — masking it would "+
+			"silence the loudest DR signal there is; got pass=%v archive=%q", e.Pass, e.SourceArchive)
+	}
+}
+
+// Two different tiers are both reported — the merge is per tier, not a single slot.
+func TestMerge_BothTiersSurvive(t *testing.T) {
+	c := mergeCollector(
+		[]RestoreTest{rt("local", "felhom-backup:backup/vzdump-lxc-9201-a.tar.zst", true,
+			time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))},
+		[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/x", true,
+			time.Date(2026, 8, 2, 5, 12, 33, 0, time.UTC))},
+	)
+	got := c.collectRestoreTests(context.Background())
+	if _, n := findTier(got, "local"); n != 1 {
+		t.Fatalf("the in-memory tier must survive the merge; got %+v", got)
+	}
+	if _, n := findTier(got, "pbs"); n != 1 {
+		t.Fatalf("the persisted tier must survive the merge; got %+v", got)
+	}
+}
+
+// A malformed timestamp must never displace a good entry — "unparseable" is not "newest".
+func TestMerge_MalformedTimestampNeverWins(t *testing.T) {
+	good := rt("pbs", "felhom-pbs:backup/ct/9201/good", true, time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC))
+	bad := RestoreTest{SourceArchive: "felhom-pbs:backup/ct/9201/bad", SourceTier: "pbs", Pass: true, TestedAt: "not-a-time"}
+
+	c := mergeCollector([]RestoreTest{good}, []RestoreTest{bad})
+	e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
+	if n != 1 || e.SourceArchive != "felhom-pbs:backup/ct/9201/good" {
+		t.Fatalf("an unparseable timestamp must not displace a good entry; got %d entr(ies), archive %q", n, e.SourceArchive)
+	}
+}
+
+// A nil proven-source leaves the pre-R-189 behaviour exactly as it was.
+func TestMerge_NilProvenSourceIsANoOp(t *testing.T) {
+	only := rt("local", "felhom-backup:backup/x.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))
+	c := mergeCollector([]RestoreTest{only}, nil)
+	got := c.collectRestoreTests(context.Background())
+	if len(got) != 1 || got[0].SourceArchive != only.SourceArchive {
+		t.Fatalf("a nil durable source must not change anything; got %+v", got)
+	}
+}