felhom-agent/internal/backup/archive_completeness_test.go

package backup

import (
	"bytes"
	"context"
	"log/slog"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// F-CRIT-2 (Campaign 8): a failed backup must not look like a fresh one.
//
// Every fixture below is a VERBATIM shape captured from the live PVE API on 2026-07-28
// (`pvesh get /nodes/<node>/storage/<store>/content`), not a hand-invented struct. That matters:
// the `unparseable` path in this package went untested for months behind a JSON shape that did not
// match production, and the whole point of this fix is that presence != validity.

// phantomEntry is the artefact a PBS daemon killed mid-upload leaves behind: listed as a restorable
// backup, 1 byte, NEWEST, and carrying no `verification`/`encrypted`/`notes` at all because it has
// no manifest (`index.json.blob` is absent on disk).
func phantomEntry() proxmox.StorageContent {
	return proxmox.StorageContent{
		VolID:   "felhom-pbs:backup/ct/9201/2026-07-28T05:31:14Z",
		Content: "backup",
		Format:  "pbs-ct",
		Size:    1,
		CTime:   1785216674,
		VMID:    9201,
	}
}

// goodPBSEntry is a real, complete offsite snapshot (demo-hp, 2026-07-28T03:40:42Z).
func goodPBSEntry() proxmox.StorageContent {
	return proxmox.StorageContent{
		VolID:   "felhom-pbs:backup/ct/9201/2026-07-28T03:40:42Z",
		Content: "backup",
		Format:  "pbs-ct",
		Size:    4353457559,
		CTime:   1785210042,
		VMID:    9201,
	}
}

// goodLocalEntry is a real, complete LOCAL vzdump (demo-hp). Note it legitimately has no
// `verification` and no `encrypted` on the wire — a dir storage has no such concept — which is
// exactly why those fields must never be used as completeness discriminators.
func goodLocalEntry() proxmox.StorageContent {
	return proxmox.StorageContent{
		VolID:   "local:backup/vzdump-lxc-9201-2026_07_28-07_29_54.tar.zst",
		Content: "backup",
		Format:  "tar.zst",
		Size:    1590431865,
		CTime:   1785216594,
		VMID:    9201,
	}
}

func runnerWithContent(t *testing.T, buf *bytes.Buffer, content []proxmox.StorageContent) *BackupRunner {
	t.Helper()
	lg := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
	return NewBackupRunner(&fakeBackupAPI{content: content}, "felhom-pbs", proxmox.ModeSnapshot, "", "", lg)
}

// Group A — the phantom must NOT set tier freshness, even though it is the newest entry.
//
// RED-PROOF: restore the old predicate in NewestArchiveTime
// (`if e.Content == "backup" && e.VMID == vmid && e.CTime > best`) → the phantom's ctime
// (1785216674) wins over the good snapshot's (1785210042) and this test fails with
// "got 1785216674, want 1785210042" — i.e. the exact F-CRIT-2 defect.
func TestNewestArchiveTime_PhantomIsNotCounted(t *testing.T) {
	var buf bytes.Buffer
	// phantom deliberately listed FIRST and is also the newest by ctime.
	r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()})

	got, found, err := r.NewestArchiveTime(context.Background(), 9201)
	if err != nil {
		t.Fatalf("NewestArchiveTime: %v", err)
	}
	if !found {
		t.Fatal("found=false — the GOOD snapshot must still be counted; rejecting everything is the thrash path")
	}
	if got.Unix() != goodPBSEntry().CTime {
		t.Errorf("freshness came from the wrong entry: got ctime %d, want %d (the good snapshot)", got.Unix(), goodPBSEntry().CTime)
	}
	if got.Unix() == phantomEntry().CTime {
		t.Error("the 1-byte manifest-less phantom set tier freshness — this is F-CRIT-2")
	}
}

// Group A — with ONLY a phantom present the tier must report "no backup", not a fresh one.
// That is what lets the controller see age_state=absent and fire its first-backup valve.
func TestNewestArchiveTime_OnlyPhantomReportsNotFound(t *testing.T) {
	var buf bytes.Buffer
	r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry()})

	_, found, err := r.NewestArchiveTime(context.Background(), 9201)
	if err != nil {
		t.Fatalf("NewestArchiveTime: %v", err)
	}
	if found {
		t.Error("found=true with only a phantom present — the tier would report fresh and go silent for a full cadence")
	}
}

// Group B — THE SCENARIO-D GUARD. A valid snapshot on EITHER tier must still be counted.
//
// This is what makes Group A safe. A filter that is too aggressive does not merely lose safety
// margin: the tier reports absent on every poll, backs up every cycle, and the R-88 breaker cannot
// save it because those backups SUCCEED. That is a continuous multi-GB write loop across the fleet.
//
// RED-PROOF: make archivePlausiblyComplete return `false, "reject everything"` unconditionally →
// both subtests fail with found=false.
func TestNewestArchiveTime_ValidSnapshotsAreStillCounted(t *testing.T) {
	for _, tc := range []struct {
		name  string
		entry proxmox.StorageContent
	}{
		{"pbs offsite (has verification+encrypted on the wire)", goodPBSEntry()},
		{"local dir vzdump (has NEITHER verification NOR encrypted — and must still count)", goodLocalEntry()},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var buf bytes.Buffer
			r := runnerWithContent(t, &buf, []proxmox.StorageContent{tc.entry})

			got, found, err := r.NewestArchiveTime(context.Background(), 9201)
			if err != nil {
				t.Fatalf("NewestArchiveTime: %v", err)
			}
			if !found {
				t.Fatalf("a REAL %s backup was rejected — this is the backup-thrash path, not extra safety", tc.name)
			}
			if got.Unix() != tc.entry.CTime {
				t.Errorf("got ctime %d, want %d", got.Unix(), tc.entry.CTime)
			}
			if strings.Contains(buf.String(), "INCOMPLETE archive") {
				t.Errorf("a valid archive was announced as incomplete:\n%s", buf.String())
			}
		})
	}
}

// Group B — the smallest REAL backup measured anywhere on the fleet (612,397,450 B, a guest-9100
// vzdump) must clear the floor with room to spare. If someone ever raises
// minPlausibleArchiveBytes past this, that is the fleet-thrash bug and this test is the tripwire.
func TestMinPlausibleArchiveBytes_LeavesHeadroomBelowTheSmallestRealBackup(t *testing.T) {
	const smallestObservedRealBackup int64 = 612397450 // fleet survey 2026-07-28
	if minPlausibleArchiveBytes >= smallestObservedRealBackup {
		t.Fatalf("floor %d B is not below the smallest real backup ever observed (%d B) — this WILL reject real archives",
			minPlausibleArchiveBytes, smallestObservedRealBackup)
	}
	if ratio := smallestObservedRealBackup / minPlausibleArchiveBytes; ratio < 100 {
		t.Errorf("floor %d B leaves only %dx headroom below the smallest real backup (%d B) — too tight",
			minPlausibleArchiveBytes, ratio, smallestObservedRealBackup)
	}
}

// Group C — UNDECIDABLE ⇒ NOT COUNTED (the fail-safe direction).
//
// A zero/absent size is not evidence of a good backup; it is absence of evidence. Erring toward
// "not fresh" costs one extra backup. Erring the other way is F-CRIT-2.
//
// RED-PROOF: flip the comparison in archivePlausiblyComplete to `e.Size > minPlausibleArchiveBytes
// || e.Size == 0` (i.e. treat unknown as complete) → the size-0 case reports ok=true and this fails.
func TestArchivePlausiblyComplete_UndecidableIsNotCounted(t *testing.T) {
	for _, tc := range []struct {
		name string
		size int64
	}{
		{"the observed phantom", 1},
		{"absent size field (unmarshals to 0)", 0},
		{"just under the floor", minPlausibleArchiveBytes - 1},
	} {
		t.Run(tc.name, func(t *testing.T) {
			e := phantomEntry()
			e.Size = tc.size
			ok, why := archivePlausiblyComplete(e)
			if ok {
				t.Errorf("size %d counted as a complete backup — undecidable must fail safe", tc.size)
			}
			if why == "" {
				t.Error("rejection carried no reason — a silent rejection is a new quiet path")
			}
		})
	}
	if ok, why := archivePlausiblyComplete(goodPBSEntry()); !ok {
		t.Errorf("a real snapshot was rejected: %s", why)
	}
}

// Group D — the rejection is announced ONCE per snapshot, not once per due-check.
//
// The due-check runs every 5 minutes and a phantom persists indefinitely (server-side prune does
// not collect it), so per-poll logging would emit ~288 identical lines a day and bury the signal.
//
// RED-PROOF: delete the `if seen { return }` guard in warnRejectedArchiveOnce → this test reports
// "logged 5 times, want 1".
func TestNewestArchiveTime_RejectionLoggedOncePerSnapshot(t *testing.T) {
	var buf bytes.Buffer
	r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()})

	const polls = 5
	for i := 0; i < polls; i++ {
		if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil {
			t.Fatalf("poll %d: %v", i, err)
		}
	}

	n := strings.Count(buf.String(), "INCOMPLETE archive")
	if n != 1 {
		t.Errorf("rejection logged %d times across %d polls, want exactly 1:\n%s", n, polls, buf.String())
	}
	out := buf.String()
	if !strings.Contains(out, phantomEntry().VolID) {
		t.Errorf("the log line does not NAME the rejected snapshot:\n%s", out)
	}
	if !strings.Contains(out, "below the") {
		t.Errorf("the log line does not say WHY it was rejected:\n%s", out)
	}
	if !strings.Contains(out, "level=WARN") {
		t.Errorf("rejection was not logged at WARN:\n%s", out)
	}
}

// Group D — a SECOND, distinct phantom is announced separately. The dedupe must be per snapshot,
// not a one-shot latch that hides every later phantom.
func TestNewestArchiveTime_DistinctPhantomsEachAnnounced(t *testing.T) {
	var buf bytes.Buffer
	second := phantomEntry()
	second.VolID = "felhom-pbs:backup/ct/9201/2026-07-29T05:31:14Z"
	second.CTime = phantomEntry().CTime + 86400
	r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), second, goodPBSEntry()})

	for i := 0; i < 3; i++ {
		if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil {
			t.Fatalf("poll %d: %v", i, err)
		}
	}
	if n := strings.Count(buf.String(), "INCOMPLETE archive"); n != 2 {
		t.Errorf("got %d rejection lines for 2 distinct phantoms across 3 polls, want 2:\n%s", n, buf.String())
	}
}

// R-99 (`09` §3 decision 140): the WARN for a PBS phantom ends with the cleanup runbook, so whoever sees it knows the
// one sanctioned way to remove it; a tiny archive on a dir storage is not a PBS phantom and gets no pointer.
// RED-PROOF: drop the `msg += phantomCleanupPointer` line → "the PBS phantom WARN does not end with the runbook pointer".
func TestRejectedArchiveWarnNamesTheCleanupRunbook(t *testing.T) {
	var buf bytes.Buffer
	r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()})
	if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil {
		t.Fatal(err)
	}
	const want = "INCOMPLETE archive when computing tier freshness — it is not a successful backup — a phantom leftover; delete it by felhom.eu documentation/runbooks/pbs-phantom-cleanup.md (09 §3 decision 140)"
	if !strings.Contains(buf.String(), want) {
		t.Errorf("the PBS phantom WARN does not end with the runbook pointer:\n%s", buf.String())
	}
	local := phantomEntry()
	local.Format, local.VolID = "tar.zst", "local:backup/vzdump-lxc-9201-2026_07_28-05_31_14.tar.zst"
	if got := rejectedArchiveMessage(local); strings.Contains(got, "pbs-phantom-cleanup") {
		t.Errorf("a dir-storage archive got the PBS runbook pointer: %s", got)
	}
}