v0.122.0: three ways the signals lied about themselves (R-189, R-188, R-186)
gates / gates (push) Successful in 7s

All three are the reporting and release path misreporting its own work. No
customer machine, no backup, no restore, no data. The restore-test itself and
when it runs are unchanged.

R-189 — a passing restore-test no longer vanishes on a restart. restore_tests[]
came only from the in-memory store, whose comment ("lost on restart; the cadence
re-populates") was true under a timer and stopped being true when R-86 made the
agent refuse to re-test a proven archive: the proof is then not repeated for a
whole archive generation. Observed live — a 14.5 GB offsite PASS reached no
host-report because the agent was restarted 2m43s later. RestoreTestState now
carries tier + verified beside the archive and renders reportable entries; the
collector merges them, one per tier, newest by TestedAt. It refuses to lie: a
record missing archive-or-tier produces no entry, and run mechanics are not
re-invented. Only successes are persisted, and the asymmetry is now written where
it will be read.

R-188 — a correct release stops emailing a failure. Only the tag PUSH moved
(build -> tag locally -> publish -> push tag): the push wakes CI, and a tag
visible before its package made the gate correctly fail a correct release about
half the time. The old order's invariant is asserted directly instead — the gate
now refuses a published version with no tag, as a bounded probe that prints its
own coverage, because the package listing api is still 401 without a token.

R-186 — a released binary can be verified by rebuilding it. -trimpath
-buildvcs=false: same source, same bytes, tag or no tag. Measured. publish-agent's
fallback also forced CGO_ENABLED=0 and produced a 74 KB different binary for the
same version; both paths now build identically. CLAUDE.md records the command.
This commit is contained in:
2026-08-03 16:40:18 +02:00
parent 3d0a1d615d
commit 7581f8140a
16 changed files with 895 additions and 42 deletions
+88 -2
View File
@@ -442,7 +442,7 @@ func TestRestoreTestState_ArchiveRoundTrips(t *testing.T) {
path := filepath.Join(t.TempDir(), "rt.json")
now := time.Now().UTC().Truncate(time.Second)
st := NewRestoreTestState(path)
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", now); err != nil {
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
t.Fatal(err)
}
re := NewRestoreTestState(path)
@@ -475,7 +475,7 @@ func TestDue_NothingDueStillNamesEveryTiersVerdict(t *testing.T) {
}}
h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
// Prove the local tier so NOTHING is due.
if err := h.st.RecordSuccess("local", "local:backup/a.tar.zst", h.clock); err != nil {
if err := h.st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", h.clock); err != nil {
t.Fatal(err)
}
@@ -505,3 +505,89 @@ func TestDue_VerdictSummaryNamesAnUnknownTier(t *testing.T) {
t.Fatalf("an unlistable tier must read as UNKNOWN with its error; got %q", got)
}
}
// ── R-189 — the persisted proof must be REPORTABLE, and must refuse to lie ───────────────────
//
// A proof held only in the in-memory store dies with the process, and under per-archive due-ness the
// agent will not repeat the work. So the persisted record has to be able to become a host-report
// entry — without inventing anything it does not know.
//
// COMPANION RED-PROOF (observed 2026-08-03): drop the `reportable()` filter from
// ProvenRestoreTests, so a pre-R-189 record (archive but no tier) is emitted →
//
// --- FAIL: TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe
// restoretest_due_test.go: a record with no TIER must not be reported (the hub keys its
// per-tier proof on it); got [{... SourceTier: ...}]
//
// Restored.
func TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe(t *testing.T) {
path := filepath.Join(t.TempDir(), "rt.json")
// v1 (a bare time), v2 (archive, no tier) and v3 (complete) side by side — every shape this
// file has ever had, which is what a real box carries after two upgrades.
legacy := `{
"old-v1": "2026-07-30T02:11:07Z",
"old-v2": {"archive":"felhom-backup:backup/vzdump-lxc-9201-a.tar.zst","proven_at":"2026-08-01T04:41:58Z"},
"felhom-pbs": {"archive":"felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z","tier":"pbs","verified":"boot+running","proven_at":"2026-08-03T13:25:14Z"}
}`
if err := writeFileForTest(path, legacy); err != nil {
t.Fatal(err)
}
got := NewRestoreTestState(path).ProvenRestoreTests(context.Background())
if len(got) != 1 {
t.Fatalf("only the record that can be described honestly may be reported; got %d: %+v", len(got), got)
}
e := got[0]
if e.SourceTier != "pbs" {
t.Fatalf("a record with no TIER must not be reported (the hub keys its per-tier proof on it); got %+v", got)
}
if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" || !e.Pass {
t.Fatalf("the reported entry must be the stored proof, unchanged; got %+v", e)
}
if e.TestedAt != "2026-08-03T13:25:14Z" {
t.Fatalf("the entry must carry the time the run passed, not now(); got %q", e.TestedAt)
}
if e.Verified != "boot+running" {
t.Fatalf("what the run verified must survive the round trip; got %q", e.Verified)
}
// Run mechanics are NOT invented: an absent duration is not a claim, a fabricated one would be.
if e.DurationSeconds != 0 || e.ScratchVMID != 0 {
t.Fatalf("the re-report must not invent run mechanics it never stored; got duration=%v scratch=%d",
e.DurationSeconds, e.ScratchVMID)
}
// The legacy records still serve the DUE-check, which is a separate question from reporting.
if _, ok := NewRestoreTestState(path).ProvenArchive("old-v2"); !ok {
t.Fatal("a v2 record must still answer the due-check even though it cannot be reported")
}
}
// A tier proved through the SCHEDULER (not by hand) lands in the state complete enough to report —
// the production path, not a hand-built fixture.
func TestScheduler_ProofIsRecordedReportably(t *testing.T) {
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, true, []string{"felhom-pbs"}, ts)
// The fake runner echoes the spec's tier; give the spec a tier the way main.go does.
h.s.spec = func(_ context.Context, archive string) reconcile.RestoreTestSpec {
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: "pbs"}
}
h.s.tick(context.Background())
got := h.st.ProvenRestoreTests(context.Background())
if len(got) != 1 {
t.Fatalf("a scheduled pass must leave a REPORTABLE proof; got %d: %+v", len(got), got)
}
if got[0].SourceTier != "pbs" || got[0].SourceArchive != "felhom-pbs:backup/ct/9201/w0" {
t.Fatalf("the proof must name the tier and the archive the run used; got %+v", got[0])
}
}
// A FAILED run leaves nothing to report — the asymmetry of §8.1, asserted rather than assumed.
func TestScheduler_AFailureLeavesNoPersistedProof(t *testing.T) {
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, false, []string{"felhom-pbs"}, ts)
h.s.tick(context.Background())
if got := h.st.ProvenRestoreTests(context.Background()); len(got) != 0 {
t.Fatalf("a FAILED run must persist nothing — a failing tier is retried, and a stored failure "+
"would outlive the fault; got %+v", got)
}
}
+99 -13
View File
@@ -1,12 +1,15 @@
package backup
import (
"context"
"encoding/json"
"os"
"path/filepath"
"sort"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)
// RestoreTestState persists the last SUCCESSFUL restore-test per backup tier.
@@ -49,16 +52,45 @@ type RestoreTestState struct {
last map[string]provenTier // target id → what was last PROVEN on that tier
}
// provenTier is one tier's proof: the archive that passed, and when it passed.
// provenTier is one tier's proof: the archive that passed, which tier it was, what was verified,
// and when.
//
// R-189 added `Tier` and `Verified`. Until then this record could answer the DUE-check but could not
// be REPORTED, and being reportable is what closes R-189: a proof held only in the in-memory result
// store vanishes on restart, and under per-archive due-ness the box will not repeat the work, so the
// hub can stay ignorant of a real success until the next archive generation.
//
// `Tier` is stored rather than derived because it is known for certain at proof time (the run's own
// spec used it to choose the restore timeout) and deriving it later would need a storage-type lookup
// at report-building time — a network call that can fail, on a path where failing means mis-labelling
// a proof. Store what you knew when you knew it.
type provenTier struct {
Archive string // volid of the archive that PASSED; "" = a legacy record with no archive
At time.Time // when that run passed (UTC)
Archive string // volid of the archive that PASSED; "" = a legacy record with no archive
Tier string // "local" | "pbs" — as the run reported it; "" = pre-R-189 record
Verified string // what the run verified (e.g. "boot+running"); "" = pre-R-189 record
At time.Time // when that run passed (UTC)
}
// provenTierJSON is the on-disk shape (R-86). The legacy shape was a bare RFC3339 STRING per
// target; both are read, only this one is written — see NewRestoreTestState.
// reportable reports whether this record can be re-reported to the hub as a restore-test result.
//
// It needs BOTH the archive and the tier: the hub keys its edge-triggered failure state on the
// archive and its per-tier proof lookup on the tier, so an entry missing either is not a usable
// proof — and emitting one anyway would be a report the hub cannot act on, dressed as evidence.
// A pre-R-189 record is therefore silently not reported; the tier's next real proof fills it in.
func (p provenTier) reportable() bool { return p.Archive != "" && p.Tier != "" }
// provenTierJSON is the on-disk shape. Two older shapes are read and neither is written:
//
// v1 (pre-R-86) "<target>": "<RFC3339>" — a time, no archive
// v2 (R-86) "<target>": {archive, proven_at} — due-check usable, not reportable
// v3 (R-189) "<target>": {archive, tier, verified, …} — both
//
// Fields absent in an older file unmarshal to "", which is exactly the "no usable proof" signal the
// readers above test for — the migration needs no version number because the absence IS the answer.
type provenTierJSON struct {
Archive string `json:"archive"`
Tier string `json:"tier,omitempty"`
Verified string `json:"verified,omitempty"`
ProvenAt string `json:"proven_at"`
}
@@ -99,21 +131,31 @@ func NewRestoreTestState(path string) *RestoreTestState {
if perr != nil {
continue
}
s.last[target] = provenTier{Archive: cur.Archive, At: t.UTC()}
s.last[target] = provenTier{Archive: cur.Archive, Tier: cur.Tier, Verified: cur.Verified, At: t.UTC()}
}
return s
}
// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed. Only call this for a
// PASSING restore-test — the archive is what makes the tier not-due, so recording one for a failed
// run would retire the archive unproven.
func (s *RestoreTestState) RecordSuccess(target, archive string, t time.Time) error {
// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed, the TIER the run
// reported, and what it verified. Only call this for a PASSING restore-test — the archive is what
// makes the tier not-due, so recording one for a failed run would retire the archive unproven.
//
// ONLY SUCCESSES ARE PERSISTED, AND THE ASYMMETRY IS DELIBERATE (R-189 §8.1). Say it here because
// the next reader will notice failures are absent and try to "fix" it:
//
// a SUCCESS suppresses future work — a proven archive is never re-tested, so a lost proof leaves
// the system quietly less tested than it believes. It must survive a restart.
//
// a FAILURE causes future work — a failing tier stays due and is retried at the next evaluation,
// so a lost failure heals itself within one interval. Persisting it would do the opposite of
// helping: a healed tier would keep reporting a failure that is no longer true.
func (s *RestoreTestState) RecordSuccess(target, archive, tier, verified string, t time.Time) error {
if target == "" {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.last[target] = provenTier{Archive: archive, At: t.UTC()}
s.last[target] = provenTier{Archive: archive, Tier: tier, Verified: verified, At: t.UTC()}
return s.saveLocked()
}
@@ -138,7 +180,13 @@ func (s *RestoreTestState) ProvenArchive(target string) (string, bool) {
return p.Archive, true
}
// Snapshot returns a copy of the last-proven TIMES — for the host-report gauge.
// Snapshot returns a copy of the last-proven TIMES.
//
// It carried the comment "for the host-report gauge" from the day it was written and **had no caller
// at all** until R-189 — a seam built and never wired, and an invariant asserted in a comment with
// nothing pinning it, in one method. The host report is now fed by ProvenRestoreTests below, which
// carries the archive and the tier that a bare timestamp cannot. This stays for callers that want
// only the times; if it acquires none, delete it rather than let it claim a purpose again.
func (s *RestoreTestState) Snapshot() map[string]time.Time {
s.mu.Lock()
defer s.mu.Unlock()
@@ -149,6 +197,41 @@ func (s *RestoreTestState) Snapshot() map[string]time.Time {
return out
}
// ProvenRestoreTests renders the persisted proofs as host-report entries — the R-189 fix.
//
// It satisfies hub.RestoreTestReporter's shape, so the collector can merge these with the in-memory
// results. What it emits is a RE-REPORT of a run that really happened, not a synthesis:
//
// - `Pass` is true because ONLY successes are stored (RecordSuccess is the sole writer);
// - `SourceArchive`, `SourceTier`, `Verified` and `TestedAt` are the values that run reported;
// - the run mechanics (scratch VMID, duration, warnings) are NOT re-invented. An absent duration
// is not a claim; a fabricated one would be.
//
// A record that cannot be reported honestly is omitted rather than padded — see provenTier.reportable.
// **A tier with no usable proof produces NO entry**: an unproven tier reading as proven would be a
// worse defect than the one this fixes.
func (s *RestoreTestState) ProvenRestoreTests(context.Context) []hub.RestoreTest {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]hub.RestoreTest, 0, len(s.last))
for _, p := range s.last {
if !p.reportable() {
continue
}
out = append(out, hub.RestoreTest{
SourceArchive: p.Archive,
SourceTier: p.Tier,
Pass: true,
Verified: p.Verified,
TestedAt: p.At.UTC().Format(time.RFC3339),
})
}
// Deterministic order: the report is compared byte-wise by the contract test, and Go's map
// iteration is randomised.
sort.Slice(out, func(i, j int) bool { return out[i].SourceTier < out[j].SourceTier })
return out
}
// OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST.
//
// This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it
@@ -185,7 +268,10 @@ func (s *RestoreTestState) OldestFirst(targets []string) []string {
func (s *RestoreTestState) saveLocked() error {
raw := make(map[string]provenTierJSON, len(s.last))
for target, p := range s.last {
raw[target] = provenTierJSON{Archive: p.Archive, ProvenAt: p.At.UTC().Format(time.RFC3339)}
raw[target] = provenTierJSON{
Archive: p.Archive, Tier: p.Tier, Verified: p.Verified,
ProvenAt: p.At.UTC().Format(time.RFC3339),
}
}
data, err := json.MarshalIndent(raw, "", " ")
if err != nil {
+5 -5
View File
@@ -303,11 +303,11 @@ func TestOldestFirst_Ordering(t *testing.T) {
t.Fatalf("unexpected: %v", got)
}
}
_ = st.RecordSuccess("local", "local:backup/a.tar.zst", now)
_ = st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", now)
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
t.Fatalf("a never-proven tier must sort before a proven one; got %v", got)
}
_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", now.Add(time.Hour))
_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", "pbs", "boot+running", now.Add(time.Hour))
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" {
t.Fatalf("the least recently proven must sort first; got %v", got)
}
@@ -318,8 +318,8 @@ func TestOldestFirst_Ordering(t *testing.T) {
func TestOldestFirst_DeterministicOnTies(t *testing.T) {
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
now := time.Now().UTC()
_ = st.RecordSuccess("b-tier", "b:archive", now)
_ = st.RecordSuccess("a-tier", "a:archive", now)
_ = st.RecordSuccess("b-tier", "b:archive", "local", "boot+running", now)
_ = st.RecordSuccess("a-tier", "a:archive", "local", "boot+running", now)
for i := 0; i < 20; i++ {
if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" {
t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got)
@@ -334,7 +334,7 @@ func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
st := NewRestoreTestState(path)
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", now); err != nil {
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
t.Fatal(err)
}
reopened := NewRestoreTestState(path)
+5 -1
View File
@@ -218,7 +218,11 @@ func (s *Scheduler) tick(ctx context.Context) {
if rt.Pass && s.rtState != nil && target != "" {
// R-86: the ARCHIVE is recorded, not merely the time — that is what makes the tier
// not-due until a NEWER archive settles, and what makes a proof survive a restart.
if err := s.rtState.RecordSuccess(target, archive, s.now()); err != nil {
// R-189: the TIER and what was VERIFIED go with it, so the proof can be RE-REPORTED after a
// restart. Both come from the run's own result, never re-derived — `rt.SourceTier` is what
// this run was actually judged as, and deriving it later would need a storage lookup that
// can fail on the one path where failing means mislabelling a proof.
if err := s.rtState.RecordSuccess(target, archive, rt.SourceTier, rt.Verified, s.now()); err != nil {
s.logger.Warn("backup: could not persist the restore-test proof state", "target", target, "err", err)
}
}
+16 -2
View File
@@ -10,8 +10,22 @@ import (
// Store holds the agent's LATEST backup result per target and the latest restore-test
// result — the point-in-time state the host-report surfaces. It is updated by the backup
// runner + the restore-test scheduler/selftest and read by the collector via the hub
// BackupReporter / RestoreTestReporter seams. In-memory (lost on restart; the cadence
// re-populates) and mutex-guarded for the concurrent collector vs scheduler access.
// BackupReporter / RestoreTestReporter seams. In-memory and mutex-guarded for the concurrent
// collector vs scheduler access.
//
// **"lost on restart; the cadence re-populates" — that sentence used to be here and it is now
// FALSE for restore-tests (R-189, 2026-08-03).** It was true while a timer re-tested every tier
// daily. Under R-86's per-archive due-check the agent will NOT re-test an archive it has already
// proven, so a proof lost to a restart is not repeated until the next archive generation — a week on
// the offsite tier — and the hub reports that tier unproven throughout. Observed, not predicted: a
// real 14.5 GB offsite restore passed, the agent was restarted 2 m 43 s later for a deploy, and two
// consecutive host-reports carried `0 restore-tests`.
//
// The durable half is `RestoreTestState` (on disk, per tier, with the archive) and the collector
// merges the two — see hub.ProvenRestoreTestReporter. This store remains the ONLY place a FAILURE is
// recorded, and that asymmetry is deliberate: a failing tier stays due and is retried, so a lost
// failure heals itself, while a lost success leaves the system quietly less tested than it believes.
// Backups are unaffected — their freshness has a ground truth on the storage (R-84).
type Store struct {
mu sync.Mutex
byTarget map[string]hub.Backup // latest backup per target id
+96 -5
View File
@@ -47,6 +47,22 @@ type RestoreTestReporter interface {
RestoreTests(ctx context.Context) []RestoreTest
}
// ProvenRestoreTestReporter is the DURABLE half of the restore-test signal (R-189).
//
// RestoreTestReporter above is backed by an in-memory store whose own comment used to read "lost on
// restart; the cadence re-populates". That was true while a timer re-tested every tier daily. It
// stopped being true on 2026-08-03: under per-archive due-ness the agent will not re-test an archive
// it has already proven, so a proof lost to a restart is not repeated for a whole archive generation
// — a week on the offsite tier — and the hub reports the tier unproven the entire time.
//
// Observed, not predicted: a real 14.5 GB offsite restore passed at 15:25:14, the agent was restarted
// 2 m 43 s later for a deploy, and the hub logged `0 restore-tests` on the next two reports.
//
// (*backup.RestoreTestState).ProvenRestoreTests satisfies this. nil → the merge is a no-op.
type ProvenRestoreTestReporter interface {
ProvenRestoreTests(ctx context.Context) []RestoreTest
}
// PBSReporter is the slice-6-Phase-B seam the pbs verify loop plugs into (same pattern).
// Returns the agent's latest-known PBS snapshot inventory + verify-state. nil → empty.
type PBSReporter interface {
@@ -79,6 +95,7 @@ type Collector struct {
storage StorageObserver
backups BackupReporter
restoreTests RestoreTestReporter
provenTests ProvenRestoreTestReporter
pbs PBSReporter
temp TempReader // slice 9: host CPU/chassis temp (nil-safe → nil temp)
capProbe func(ctx context.Context) []capability.Status // v0.44.0: privileged-capability self-check (nil → empty)
@@ -427,16 +444,90 @@ func (c *Collector) collectBackups(ctx context.Context) []Backup {
return []Backup{}
}
// collectRestoreTests merges the in-memory result with the PERSISTED per-tier proofs (R-189).
//
// The rule is ONE ENTRY PER TIER, NEWEST WINS, and it falls out of what each source means rather
// than from a preference between them:
//
// - the in-memory store holds this process's latest run, pass OR fail. A failure exists nowhere
// else and must always reach the hub — a failing tier is retried at the next evaluation, so its
// record is short-lived by design;
// - the persisted state holds the last SUCCESS per tier and survives a restart.
//
// Comparing by TestedAt gives the right answer in every case without special-casing: a fresh failure
// beats an older stored success (the failure is the news), a stored success beats a stale in-memory
// entry after a restart, and a tier proved twice never appears twice — two entries for one tier would
// read at the hub as two tests.
//
// A tier with no usable persisted proof contributes NOTHING. Reporting an unproven tier as proven
// would be a worse defect than the one this closes.
func (c *Collector) collectRestoreTests(ctx context.Context) []RestoreTest {
if c.restoreTests == nil {
return []RestoreTest{}
out := []RestoreTest{}
if c.restoreTests != nil {
if r := c.restoreTests.RestoreTests(ctx); r != nil {
out = append(out, r...)
}
}
if r := c.restoreTests.RestoreTests(ctx); r != nil {
return r
if c.provenTests == nil {
return out
}
return []RestoreTest{}
// Index what we already have by tier, keeping the newest per tier.
best := map[string]int{} // tier → index into out
for i, rt := range out {
if rt.SourceTier == "" {
continue // untiered entry: never deduped, never overwritten — we cannot say what it is
}
if j, seen := best[rt.SourceTier]; !seen || newerRestoreTest(rt, out[j]) {
best[rt.SourceTier] = i
}
}
for _, p := range c.provenTests.ProvenRestoreTests(ctx) {
if p.SourceTier == "" {
continue // not usable as a per-tier proof; the state layer already filters these
}
i, seen := best[p.SourceTier]
if !seen {
out = append(out, p)
best[p.SourceTier] = len(out) - 1
continue
}
if newerRestoreTest(p, out[i]) {
out[i] = p
}
}
return out
}
// newerRestoreTest reports whether a was tested after b. An unparseable or absent timestamp is
// treated as OLDER, so a malformed entry can never displace a good one.
func newerRestoreTest(a, b RestoreTest) bool {
ta, aok := parseRestoreTestedAt(a.TestedAt)
tb, bok := parseRestoreTestedAt(b.TestedAt)
if !aok {
return false
}
if !bok {
return true
}
return ta.After(tb)
}
func parseRestoreTestedAt(s string) (time.Time, bool) {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return time.Time{}, false
}
return t.UTC(), true
}
// SetProvenRestoreTests wires the durable proof source. It is a setter rather than a constructor
// argument because the persisted state is opened later in main() than the collector is built; the
// same shape as the other late-wired seams here. **The wiring is asserted by an AST test** — the
// method it feeds carried a doc comment naming a "host-report gauge" for weeks with no caller at
// all, and this fix must not become the next instance of that.
func (c *Collector) SetProvenRestoreTests(p ProvenRestoreTestReporter) { c.provenTests = p }
// collectPBSSnapshots reads the latest PBS snapshot inventory via the seam (nil → empty).
func (c *Collector) collectPBSSnapshots(ctx context.Context) []PBSSnapshot {
if c.pbs == nil {
+213
View File
@@ -0,0 +1,213 @@
package hub
import (
"context"
"testing"
"time"
)
// R-189 — a passing restore-test must survive an agent restart and reach the hub.
//
// THE OBSERVATION THIS EXISTS FOR (2026-08-03, demo-felhom): a real 14.5 GB offsite restore-test
// PASSED at 15:25:14; the agent was restarted 2 m 43 s later for a deploy; the hub logged
// `0 restore-tests` on the next two host-reports. The in-memory store's own comment said "lost on
// restart; the cadence re-populates", which was true under a timer and stopped being true when R-86
// made the agent refuse to re-test an archive it has already proven.
//
// Timestamps here carry JITTER (odd minutes and seconds, not round hours) — yesterday a test was
// hollow because a perfectly regular series landed exactly on a threshold and passed under the
// mutation it was meant to catch.
type fakeLatest struct{ tests []RestoreTest }
func (f *fakeLatest) RestoreTests(context.Context) []RestoreTest { return f.tests }
type fakeProven struct{ tests []RestoreTest }
func (f *fakeProven) ProvenRestoreTests(context.Context) []RestoreTest { return f.tests }
func rt(tier, archive string, pass bool, at time.Time) RestoreTest {
return RestoreTest{
SourceArchive: archive, SourceTier: tier, Pass: pass,
Verified: "boot+running", TestedAt: at.UTC().Format(time.RFC3339),
}
}
// mergeCollector builds a Collector with only the two restore-test seams wired — the merge is what
// is under test, not the rest of the collection.
func mergeCollector(latest, proven []RestoreTest) *Collector {
c := &Collector{}
if latest != nil {
c.restoreTests = &fakeLatest{tests: latest}
}
if proven != nil {
c.provenTests = &fakeProven{tests: proven}
}
return c
}
func findTier(got []RestoreTest, tier string) (RestoreTest, int) {
var hit RestoreTest
n := 0
for _, e := range got {
if e.SourceTier == tier {
hit, n = e, n+1
}
}
return hit, n
}
// ── SCENARIO A — a proof survives a restart and reaches the hub ──────────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-03): delete the `c.provenTests` merge from
// collectRestoreTests (return the in-memory slice as it used to) →
//
// --- FAIL: TestMerge_ProofSurvivesARestart
// restoretest_merge_test.go: after a restart the persisted proof must be reported; got 0 entr(ies)
//
// which is exactly the live observation: `0 restore-tests`. Restored.
func TestMerge_ProofSurvivesARestart(t *testing.T) {
provenAt := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) // the real run's timestamp
// After a restart the in-memory store is EMPTY — this is the whole point.
c := mergeCollector([]RestoreTest{}, []RestoreTest{
rt("pbs", "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z", true, provenAt),
})
got := c.collectRestoreTests(context.Background())
if len(got) != 1 {
t.Fatalf("after a restart the persisted proof must be reported; got %d entr(ies): %+v", len(got), got)
}
e := got[0]
if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" {
t.Fatalf("the entry must name the archive that was proven — the hub keys on it; got %q", e.SourceArchive)
}
if e.SourceTier != "pbs" || !e.Pass {
t.Fatalf("the entry must be a PASS on the tier it was proven on; got tier=%q pass=%v", e.SourceTier, e.Pass)
}
if e.TestedAt != provenAt.Format(time.RFC3339) {
t.Fatalf("the entry must carry the ORIGINAL test time, not now(); got %q", e.TestedAt)
}
}
// ── SCENARIO B — the report does not invent a pass ───────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): make the state layer emit an entry for an unproven tier (drop the
// `reportable()` filter in ProvenRestoreTests, so a legacy record with no archive is emitted) — the
// equivalent at this layer is a proven-source that returns an entry for a tier nothing proved, which
// this test injects directly and the assertion below rejects.
func TestMerge_NeverInventsAPassForAnUnprovenTier(t *testing.T) {
// Nothing proven anywhere: no in-memory result, no persisted proof.
c := mergeCollector([]RestoreTest{}, []RestoreTest{})
if got := c.collectRestoreTests(context.Background()); len(got) != 0 {
t.Fatalf("a tier with no proof must produce NO entry — an unproven tier reading as proven is "+
"worse than the defect being fixed; got %+v", got)
}
// And an entry the state layer could not describe (no tier) is never promoted into a proof.
c2 := mergeCollector([]RestoreTest{}, []RestoreTest{
{SourceArchive: "local:backup/x.tar.zst", SourceTier: "", Pass: true,
TestedAt: time.Date(2026, 8, 1, 4, 41, 58, 0, time.UTC).Format(time.RFC3339)},
})
if got := c2.collectRestoreTests(context.Background()); len(got) != 0 {
t.Fatalf("a persisted record with no tier is not a usable proof and must be dropped; got %+v", got)
}
}
// ── SCENARIO C — a fresh in-memory result wins, and never duplicates ─────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-03): remove the de-duplication (append every persisted entry
// unconditionally) →
//
// --- FAIL: TestMerge_NewerWinsAndNeverDuplicatesATier
// restoretest_merge_test.go: one entry per tier; got 2 for "pbs" — the hub would read two tests
//
// Restored.
func TestMerge_NewerWinsAndNeverDuplicatesATier(t *testing.T) {
lastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) // jittered, from the real box
fiveMinAgo := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC)
c := mergeCollector(
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
)
got := c.collectRestoreTests(context.Background())
e, n := findTier(got, "pbs")
if n != 1 {
t.Fatalf("one entry per tier; got %d for \"pbs\" — the hub would read two tests: %+v", n, got)
}
if e.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
t.Fatalf("the NEWER result must win; got %q tested %q", e.SourceArchive, e.TestedAt)
}
// ...and the older-in-memory / newer-persisted direction, which is the post-restart case.
c2 := mergeCollector(
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
)
e2, n2 := findTier(c2.collectRestoreTests(context.Background()), "pbs")
if n2 != 1 || e2.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
t.Fatalf("newest must win regardless of which source it came from; got %d entr(ies), archive %q", n2, e2.SourceArchive)
}
}
// ── SCENARIO D — a failure still reaches the hub ─────────────────────────────────────────────
//
// The merge must not mask a failure with an older stored success. A failing tier is retried at the
// next evaluation and its record lives ONLY in memory, so losing it here would silence the loudest
// DR signal this system produces.
func TestMerge_AFailureIsStillReported(t *testing.T) {
provenLastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC)
failedJustNow := time.Date(2026, 8, 3, 13, 41, 7, 0, time.UTC)
c := mergeCollector(
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", false, failedJustNow)},
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, provenLastWeek)},
)
e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
if n != 1 {
t.Fatalf("one entry per tier; got %d: %+v", n, c.collectRestoreTests(context.Background()))
}
if e.Pass {
t.Fatalf("a FAILURE newer than the stored proof must be what is reported — masking it would "+
"silence the loudest DR signal there is; got pass=%v archive=%q", e.Pass, e.SourceArchive)
}
}
// Two different tiers are both reported — the merge is per tier, not a single slot.
func TestMerge_BothTiersSurvive(t *testing.T) {
c := mergeCollector(
[]RestoreTest{rt("local", "felhom-backup:backup/vzdump-lxc-9201-a.tar.zst", true,
time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))},
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/x", true,
time.Date(2026, 8, 2, 5, 12, 33, 0, time.UTC))},
)
got := c.collectRestoreTests(context.Background())
if _, n := findTier(got, "local"); n != 1 {
t.Fatalf("the in-memory tier must survive the merge; got %+v", got)
}
if _, n := findTier(got, "pbs"); n != 1 {
t.Fatalf("the persisted tier must survive the merge; got %+v", got)
}
}
// A malformed timestamp must never displace a good entry — "unparseable" is not "newest".
func TestMerge_MalformedTimestampNeverWins(t *testing.T) {
good := rt("pbs", "felhom-pbs:backup/ct/9201/good", true, time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC))
bad := RestoreTest{SourceArchive: "felhom-pbs:backup/ct/9201/bad", SourceTier: "pbs", Pass: true, TestedAt: "not-a-time"}
c := mergeCollector([]RestoreTest{good}, []RestoreTest{bad})
e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
if n != 1 || e.SourceArchive != "felhom-pbs:backup/ct/9201/good" {
t.Fatalf("an unparseable timestamp must not displace a good entry; got %d entr(ies), archive %q", n, e.SourceArchive)
}
}
// A nil proven-source leaves the pre-R-189 behaviour exactly as it was.
func TestMerge_NilProvenSourceIsANoOp(t *testing.T) {
only := rt("local", "felhom-backup:backup/x.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))
c := mergeCollector([]RestoreTest{only}, nil)
got := c.collectRestoreTests(context.Background())
if len(got) != 1 || got[0].SourceArchive != only.SourceArchive {
t.Fatalf("a nil durable source must not change anything; got %+v", got)
}
}