package backup import ( "bytes" "context" "log/slog" "strings" "testing" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // F-CRIT-2 (Campaign 8): a failed backup must not look like a fresh one. // // Every fixture below is a VERBATIM shape captured from the live PVE API on 2026-07-28 // (`pvesh get /nodes//storage//content`), not a hand-invented struct. That matters: // the `unparseable` path in this package went untested for months behind a JSON shape that did not // match production, and the whole point of this fix is that presence != validity. // phantomEntry is the artefact a PBS daemon killed mid-upload leaves behind: listed as a restorable // backup, 1 byte, NEWEST, and carrying no `verification`/`encrypted`/`notes` at all because it has // no manifest (`index.json.blob` is absent on disk). func phantomEntry() proxmox.StorageContent { return proxmox.StorageContent{ VolID: "felhom-pbs:backup/ct/9201/2026-07-28T05:31:14Z", Content: "backup", Format: "pbs-ct", Size: 1, CTime: 1785216674, VMID: 9201, } } // goodPBSEntry is a real, complete offsite snapshot (demo-hp, 2026-07-28T03:40:42Z). func goodPBSEntry() proxmox.StorageContent { return proxmox.StorageContent{ VolID: "felhom-pbs:backup/ct/9201/2026-07-28T03:40:42Z", Content: "backup", Format: "pbs-ct", Size: 4353457559, CTime: 1785210042, VMID: 9201, } } // goodLocalEntry is a real, complete LOCAL vzdump (demo-hp). Note it legitimately has no // `verification` and no `encrypted` on the wire — a dir storage has no such concept — which is // exactly why those fields must never be used as completeness discriminators. func goodLocalEntry() proxmox.StorageContent { return proxmox.StorageContent{ VolID: "local:backup/vzdump-lxc-9201-2026_07_28-07_29_54.tar.zst", Content: "backup", Format: "tar.zst", Size: 1590431865, CTime: 1785216594, VMID: 9201, } } func runnerWithContent(t *testing.T, buf *bytes.Buffer, content []proxmox.StorageContent) *BackupRunner { t.Helper() lg := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) return NewBackupRunner(&fakeBackupAPI{content: content}, "felhom-pbs", proxmox.ModeSnapshot, "", "", lg) } // Group A — the phantom must NOT set tier freshness, even though it is the newest entry. // // RED-PROOF: restore the old predicate in NewestArchiveTime // (`if e.Content == "backup" && e.VMID == vmid && e.CTime > best`) → the phantom's ctime // (1785216674) wins over the good snapshot's (1785210042) and this test fails with // "got 1785216674, want 1785210042" — i.e. the exact F-CRIT-2 defect. func TestNewestArchiveTime_PhantomIsNotCounted(t *testing.T) { var buf bytes.Buffer // phantom deliberately listed FIRST and is also the newest by ctime. r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()}) got, found, err := r.NewestArchiveTime(context.Background(), 9201) if err != nil { t.Fatalf("NewestArchiveTime: %v", err) } if !found { t.Fatal("found=false — the GOOD snapshot must still be counted; rejecting everything is the thrash path") } if got.Unix() != goodPBSEntry().CTime { t.Errorf("freshness came from the wrong entry: got ctime %d, want %d (the good snapshot)", got.Unix(), goodPBSEntry().CTime) } if got.Unix() == phantomEntry().CTime { t.Error("the 1-byte manifest-less phantom set tier freshness — this is F-CRIT-2") } } // Group A — with ONLY a phantom present the tier must report "no backup", not a fresh one. // That is what lets the controller see age_state=absent and fire its first-backup valve. func TestNewestArchiveTime_OnlyPhantomReportsNotFound(t *testing.T) { var buf bytes.Buffer r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry()}) _, found, err := r.NewestArchiveTime(context.Background(), 9201) if err != nil { t.Fatalf("NewestArchiveTime: %v", err) } if found { t.Error("found=true with only a phantom present — the tier would report fresh and go silent for a full cadence") } } // Group B — THE SCENARIO-D GUARD. A valid snapshot on EITHER tier must still be counted. // // This is what makes Group A safe. A filter that is too aggressive does not merely lose safety // margin: the tier reports absent on every poll, backs up every cycle, and the R-88 breaker cannot // save it because those backups SUCCEED. That is a continuous multi-GB write loop across the fleet. // // RED-PROOF: make archivePlausiblyComplete return `false, "reject everything"` unconditionally → // both subtests fail with found=false. func TestNewestArchiveTime_ValidSnapshotsAreStillCounted(t *testing.T) { for _, tc := range []struct { name string entry proxmox.StorageContent }{ {"pbs offsite (has verification+encrypted on the wire)", goodPBSEntry()}, {"local dir vzdump (has NEITHER verification NOR encrypted — and must still count)", goodLocalEntry()}, } { t.Run(tc.name, func(t *testing.T) { var buf bytes.Buffer r := runnerWithContent(t, &buf, []proxmox.StorageContent{tc.entry}) got, found, err := r.NewestArchiveTime(context.Background(), 9201) if err != nil { t.Fatalf("NewestArchiveTime: %v", err) } if !found { t.Fatalf("a REAL %s backup was rejected — this is the backup-thrash path, not extra safety", tc.name) } if got.Unix() != tc.entry.CTime { t.Errorf("got ctime %d, want %d", got.Unix(), tc.entry.CTime) } if strings.Contains(buf.String(), "INCOMPLETE archive") { t.Errorf("a valid archive was announced as incomplete:\n%s", buf.String()) } }) } } // Group B — the smallest REAL backup measured anywhere on the fleet (612,397,450 B, a guest-9100 // vzdump) must clear the floor with room to spare. If someone ever raises // minPlausibleArchiveBytes past this, that is the fleet-thrash bug and this test is the tripwire. func TestMinPlausibleArchiveBytes_LeavesHeadroomBelowTheSmallestRealBackup(t *testing.T) { const smallestObservedRealBackup int64 = 612397450 // fleet survey 2026-07-28 if minPlausibleArchiveBytes >= smallestObservedRealBackup { t.Fatalf("floor %d B is not below the smallest real backup ever observed (%d B) — this WILL reject real archives", minPlausibleArchiveBytes, smallestObservedRealBackup) } if ratio := smallestObservedRealBackup / minPlausibleArchiveBytes; ratio < 100 { t.Errorf("floor %d B leaves only %dx headroom below the smallest real backup (%d B) — too tight", minPlausibleArchiveBytes, ratio, smallestObservedRealBackup) } } // Group C — UNDECIDABLE ⇒ NOT COUNTED (the fail-safe direction). // // A zero/absent size is not evidence of a good backup; it is absence of evidence. Erring toward // "not fresh" costs one extra backup. Erring the other way is F-CRIT-2. // // RED-PROOF: flip the comparison in archivePlausiblyComplete to `e.Size > minPlausibleArchiveBytes // || e.Size == 0` (i.e. treat unknown as complete) → the size-0 case reports ok=true and this fails. func TestArchivePlausiblyComplete_UndecidableIsNotCounted(t *testing.T) { for _, tc := range []struct { name string size int64 }{ {"the observed phantom", 1}, {"absent size field (unmarshals to 0)", 0}, {"just under the floor", minPlausibleArchiveBytes - 1}, } { t.Run(tc.name, func(t *testing.T) { e := phantomEntry() e.Size = tc.size ok, why := archivePlausiblyComplete(e) if ok { t.Errorf("size %d counted as a complete backup — undecidable must fail safe", tc.size) } if why == "" { t.Error("rejection carried no reason — a silent rejection is a new quiet path") } }) } if ok, why := archivePlausiblyComplete(goodPBSEntry()); !ok { t.Errorf("a real snapshot was rejected: %s", why) } } // Group D — the rejection is announced ONCE per snapshot, not once per due-check. // // The due-check runs every 5 minutes and a phantom persists indefinitely (server-side prune does // not collect it), so per-poll logging would emit ~288 identical lines a day and bury the signal. // // RED-PROOF: delete the `if seen { return }` guard in warnRejectedArchiveOnce → this test reports // "logged 5 times, want 1". func TestNewestArchiveTime_RejectionLoggedOncePerSnapshot(t *testing.T) { var buf bytes.Buffer r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()}) const polls = 5 for i := 0; i < polls; i++ { if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil { t.Fatalf("poll %d: %v", i, err) } } n := strings.Count(buf.String(), "INCOMPLETE archive") if n != 1 { t.Errorf("rejection logged %d times across %d polls, want exactly 1:\n%s", n, polls, buf.String()) } out := buf.String() if !strings.Contains(out, phantomEntry().VolID) { t.Errorf("the log line does not NAME the rejected snapshot:\n%s", out) } if !strings.Contains(out, "below the") { t.Errorf("the log line does not say WHY it was rejected:\n%s", out) } if !strings.Contains(out, "level=WARN") { t.Errorf("rejection was not logged at WARN:\n%s", out) } } // Group D — a SECOND, distinct phantom is announced separately. The dedupe must be per snapshot, // not a one-shot latch that hides every later phantom. func TestNewestArchiveTime_DistinctPhantomsEachAnnounced(t *testing.T) { var buf bytes.Buffer second := phantomEntry() second.VolID = "felhom-pbs:backup/ct/9201/2026-07-29T05:31:14Z" second.CTime = phantomEntry().CTime + 86400 r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), second, goodPBSEntry()}) for i := 0; i < 3; i++ { if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil { t.Fatalf("poll %d: %v", i, err) } } if n := strings.Count(buf.String(), "INCOMPLETE archive"); n != 2 { t.Errorf("got %d rejection lines for 2 distinct phantoms across 3 polls, want 2:\n%s", n, buf.String()) } }