felhom-controller/controller/internal/backup/r87_proof_job_test.go

package backup

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// R-87 Group B — the JOB.
//
// Everything here drives the real `ProveOffboxUnit` through the two seams REUSE.md names:
// `SetOffboxRunner` (the restic exec seam — deliberately NOT a `resticStepFn`, so the
// `unlock --remove-all` escalation stays visible to the argv assertions) and
// `SetOffboxLatestSnapshotFn`. No restic, no ssh, no docker.

// proofHarness is a Manager wired for the proof job, with a recorder for every restic argv.
type proofHarness struct {
	m    *Manager
	sett *settings.Settings
	prov *offbox3aProvider
	// drive is the registered storage path everything is resolved under.
	drive string

	mu   sync.Mutex
	argv [][]string
	// unitFor decides what the fake restore materialises for a stack; nil = a healthy unit.
	unitFor map[string]unitFixture
	// snapFor overrides the snapshot id per stack; "" entries mean "no snapshot".
	snapFor map[string]string
	// failRestore makes the fake restic return an error.
	failRestore bool
}

// unitPathFor is the absolute path a snapshot records for a stack's recovery unit — the shape
// `offboxUnitPathOf` matches on.
func unitPathFor(stack string) string {
	return "/mnt/sys_drive/felhom-data/backups/primary/" + stack
}

func newProofHarness(t *testing.T, stacks ...string) *proofHarness {
	t.Helper()
	m, sett := newOffboxManager(t)
	drive := t.TempDir()
	if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "data", Schedulable: true}); err != nil {
		t.Fatal(err)
	}
	deployed := map[string]bool{}
	for _, s := range stacks {
		deployed[s] = true
	}
	prov := &offbox3aProvider{
		hdd: map[string]string{}, binds: map[string][]ClassifiedBind{},
		has: map[string]bool{}, deployed: deployed,
	}
	m.SetStackProvider(prov)

	h := &proofHarness{m: m, sett: sett, prov: prov, drive: drive,
		unitFor: map[string]unitFixture{}, snapFor: map[string]string{}}

	m.SetOffboxLatestSnapshotFn(func(_ context.Context, stack string) (string, []string, error) {
		id, ok := h.snapFor[stack]
		if !ok {
			id = "snap-" + stack
		}
		if id == "" {
			return "", nil, os.ErrNotExist
		}
		return id, []string{unitPathFor(stack)}, nil
	})

	// The fake restic: records the argv, and for a `restore` MATERIALISES the unit the test asked
	// for, at the path inside --target that a real restic would write it to.
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		h.mu.Lock()
		h.argv = append(h.argv, append([]string{}, args...))
		h.mu.Unlock()
		if len(args) == 0 || !containsArg(args, "restore") {
			return nil, nil
		}
		if h.failRestore {
			return []byte("simulated restic failure"), os.ErrPermission
		}
		target, include := argValue(args, "--target"), argValue(args, "--include")
		if target == "" || include == "" {
			return nil, nil
		}
		stack := filepath.Base(include)
		dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
		materialiseUnit(t, dest, h.unitFor[stack])
		return nil, nil
	})
	// Plenty of headroom unless a test says otherwise.
	m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
	return h
}

// containsArg lives in r358_scratch_marker_test.go — the same question, one implementation.

func argValue(args []string, flag string) string {
	for i, a := range args {
		if a == flag && i+1 < len(args) {
			return args[i+1]
		}
	}
	return ""
}

// materialiseUnit writes a recovery unit at dest, mirroring what a restore would leave behind.
// The zero fixture is a HEALTHY database app.
func materialiseUnit(t *testing.T, dest string, f unitFixture) {
	t.Helper()
	if f.compose == "" && len(f.dbDumps) == 0 && len(f.volumeDumps) == 0 && !f.noManifest && f.rawManifest == "" {
		f = unitFixture{
			compose:     composeWithDB,
			dbDumps:     []string{"app-mariadb.sql"},
			volumeDumps: []string{"a.tar", "b.tar"},
		}
	}
	for _, sub := range []string{"compose", "db-dumps", "volume-dumps"} {
		if err := os.MkdirAll(filepath.Join(dest, sub), 0o755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
	}
	if f.compose != "" {
		if err := os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte(f.compose), 0o644); err != nil {
			t.Fatalf("compose: %v", err)
		}
	}
	if !f.declareOnly {
		for sub, names := range map[string][]string{"db-dumps": f.dbDumps, "volume-dumps": f.volumeDumps} {
			for _, n := range names {
				if err := os.WriteFile(filepath.Join(dest, sub, n), []byte("x"), 0o644); err != nil {
					t.Fatalf("write: %v", err)
				}
			}
		}
	}
	if f.noManifest {
		return
	}
	b, _ := json.Marshal(RecoveryManifest{DBDumps: f.dbDumps, VolumeDumps: f.volumeDumps})
	if f.rawManifest != "" {
		b = []byte(f.rawManifest)
	}
	if err := os.WriteFile(filepath.Join(dest, "manifest.json"), b, 0o644); err != nil {
		t.Fatalf("manifest: %v", err)
	}
}

func (h *proofHarness) allArgs() [][]string {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.argv
}

func (h *proofHarness) proofScratch(t *testing.T, stack string) string {
	t.Helper()
	s, _, err := h.m.offboxProofScratchDir(stack)
	if err != nil {
		t.Fatalf("proof scratch dir: %v", err)
	}
	return s
}

// ── B1 ───────────────────────────────────────────────────────────────────────────────────────────

// TestR87_SkipsWhenRunningFlagHeld asserts a NON-EFFECT, the way TestR359_SkipsWhenRunningFlagHeld
// does: restic was never invoked at all, and due-ness did not move.
func TestR87_SkipsWhenRunningFlagHeld(t *testing.T) {
	h := newProofHarness(t, "kimai")
	if err := h.m.AcquireRunningForTest(); err != nil {
		t.Fatalf("could not take the flag: %v", err)
	}
	res := h.m.ProveOffboxUnit(context.Background())
	if !res.Skipped {
		t.Fatalf("must SKIP while the single-writer flag is held; got %+v", res)
	}
	if got := len(h.allArgs()); got != 0 {
		t.Fatalf("a skip must invoke restic ZERO times; got %d invocations: %v", got, h.allArgs())
	}
	if res.Verdict() != "" {
		t.Fatalf("a skip reaches no verdict; got %q", res.Verdict())
	}
	h.m.RecordProofVerdict(res)
	if tt := h.sett.GetOffboxTarget(); len(tt.ProvedSnapshots) != 0 || tt.LastProofResult != "" {
		t.Fatalf("a skip must NOT advance due-ness; got proved=%v result=%q", tt.ProvedSnapshots, tt.LastProofResult)
	}
}

// ── B2 ───────────────────────────────────────────────────────────────────────────────────────────

// TestR87_ScratchIsDeletedOnEveryPath — pass, fail, cannot-judge and a restore error.
//
// RED-PROOF (run 2026-08-31): removing the `defer m.removeProofScratch(...)` makes the pass and fail
// rows fail with the scratch directory still present.
func TestR87_ScratchIsDeletedOnEveryPath(t *testing.T) {
	cases := map[string]struct {
		fixture     unitFixture
		failRestore bool
	}{
		"pass":          {fixture: unitFixture{}},
		"fail empty":    {fixture: unitFixture{compose: composeWithDB}},
		"cannot judge":  {fixture: unitFixture{noManifest: false, compose: "", dbDumps: []string{"d.sql"}}},
		"restore error": {failRestore: true},
	}
	for name, c := range cases {
		t.Run(name, func(t *testing.T) {
			h := newProofHarness(t, "kimai")
			h.unitFor["kimai"] = c.fixture
			h.failRestore = c.failRestore
			scratch := h.proofScratch(t, "kimai")

			h.m.ProveOffboxUnit(context.Background())

			if _, err := os.Stat(scratch); !os.IsNotExist(err) {
				t.Fatalf("the proof copy must be gone on EVERY path; %s still exists (stat err=%v)", scratch, err)
			}
		})
	}
}

// TestR87_ProofScratchIsNotTheCustomerVerificationCopy — the delete above must never be able to reach
// a copy the CUSTOMER made. Different roots is how that is guaranteed rather than hoped.
func TestR87_ProofScratchIsNotTheCustomerVerificationCopy(t *testing.T) {
	h := newProofHarness(t, "kimai")
	proof := h.proofScratch(t, "kimai")
	customer, _, err := h.m.offboxRestoreScratchDir("kimai")
	if err != nil {
		t.Fatal(err)
	}
	if proof == customer {
		t.Fatal("the proof copy and the customer's verification copy MUST NOT share a path — the nightly delete would destroy the customer's copy")
	}
	if !strings.Contains(proof, "offsite-proof") || !strings.Contains(customer, "offsite-restore") {
		t.Fatalf("roots are not the expected pair: proof=%q customer=%q", proof, customer)
	}

	// And the customer's copy survives a full proof run that deletes its own.
	if err := os.MkdirAll(customer, 0o755); err != nil {
		t.Fatal(err)
	}
	sentinel := filepath.Join(customer, "customer-copy-marker")
	if err := os.WriteFile(sentinel, []byte("keep me"), 0o644); err != nil {
		t.Fatal(err)
	}
	h.m.ProveOffboxUnit(context.Background())
	if _, err := os.Stat(sentinel); err != nil {
		t.Fatalf("the nightly proof deleted the CUSTOMER's verification copy: %v", err)
	}
}

// ── B3 ───────────────────────────────────────────────────────────────────────────────────────────

// TestR87_NeverWritesToTheRepository is Scenario E, asserted as a NON-EFFECT on the argv rather than
// as the absence of an error.
//
// RED-PROOF (run 2026-08-31): routing the restore through `resticStep` with the `unlockStale`
// pre-flight — i.e. what `RestoreOffboxScratch` does — makes this fail on `unlock` appearing in the
// argv and on `--no-lock` being absent.
func TestR87_NeverWritesToTheRepository(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.m.ProveOffboxUnit(context.Background())

	all := h.allArgs()
	if len(all) == 0 {
		t.Fatal("the proof must have invoked restic — a zero-invocation run proves nothing about the argv")
	}
	// Every write verb restic has that this codebase ever issues.
	for _, args := range all {
		for _, verb := range []string{"unlock", "forget", "prune", "backup", "init", "--remove-all"} {
			if containsArg(args, verb) {
				t.Fatalf("the proof issued a WRITE verb %q against the repository: %v", verb, args)
			}
		}
	}
	var sawRestore bool
	for _, args := range all {
		if containsArg(args, "restore") {
			sawRestore = true
			if !containsArg(args, "--no-lock") {
				t.Fatalf("the proof restore must carry --no-lock so no lock file can be created: %v", args)
			}
		}
	}
	if !sawRestore {
		t.Fatal("no restore was issued — the --no-lock assertion above never ran")
	}
}

// ── B4, B5, B6 — due-ness ────────────────────────────────────────────────────────────────────────

func TestR87_OneAppPerRun(t *testing.T) {
	h := newProofHarness(t, "bookstack", "docmost", "kimai")
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Stack == "" {
		t.Fatal("a run must pick an app")
	}
	var restores int
	for _, args := range h.allArgs() {
		if containsArg(args, "restore") {
			restores++
		}
	}
	if restores != 1 {
		t.Fatalf("one app per run: want exactly 1 restore, got %d", restores)
	}
}

// TestR87_ProvedSnapshotIsRecordedNotATimestamp.
//
// RED-PROOF (run 2026-08-31): recording `time.Now()` in `ProvedSnapshots[stack]` instead of the
// snapshot ID makes this fail on the stored value, and makes B6 fail too because the app never
// becomes due again.
func TestR87_ProvedSnapshotIsRecordedNotATimestamp(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.snapFor["kimai"] = "a07c36a1"
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Verdict() != string(UnitProofPass) {
		t.Fatalf("fixture should pass; got %q (%q)", res.Verdict(), res.Judgement.Reason)
	}
	h.m.RecordProofVerdict(res)

	tt := h.sett.GetOffboxTarget()
	if got := tt.ProvedSnapshots["kimai"]; got != "a07c36a1" {
		t.Fatalf("the SNAPSHOT ID must be recorded; got %q", got)
	}
	if tt.LastProofSnapshot != "a07c36a1" || tt.LastProofStack != "kimai" || tt.LastProofResult != "pass" {
		t.Fatalf("the verdict record is wrong: %+v", tt)
	}
	// The record must not be a time: a timestamp would parse as one and an ID must not.
	if strings.Contains(tt.ProvedSnapshots["kimai"], ":") || strings.Contains(tt.ProvedSnapshots["kimai"], "T") {
		t.Fatalf("ProvedSnapshots looks like a timestamp, not a snapshot ID: %q", tt.ProvedSnapshots["kimai"])
	}
}

func TestR87_AlreadyProvedSnapshotIsNotReProved(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.snapFor["kimai"] = "same-id"
	h.m.RecordProofVerdict(ProofResult{Stack: "kimai", SnapshotID: "same-id",
		Judgement: UnitProofResult{Verdict: UnitProofPass}})

	res := h.m.ProveOffboxUnit(context.Background())
	if !res.NoSnapshot {
		t.Fatalf("an already-proved newest snapshot must leave nothing due; got stack=%q", res.Stack)
	}
	for _, args := range h.allArgs() {
		if containsArg(args, "restore") {
			t.Fatalf("nothing due must download nothing; got %v", args)
		}
	}
}

// TestR87_NewSnapshotMakesAProvedAppDueAgain — the half a timestamp cannot do.
func TestR87_NewSnapshotMakesAProvedAppDueAgain(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.snapFor["kimai"] = "old-id"
	h.m.RecordProofVerdict(ProofResult{Stack: "kimai", SnapshotID: "old-id",
		Judgement: UnitProofResult{Verdict: UnitProofPass}})

	h.snapFor["kimai"] = "new-id" // the nightly backup landed
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Stack != "kimai" || res.SnapshotID != "new-id" {
		t.Fatalf("a NEW snapshot must make the app due again; got stack=%q snapshot=%q noSnapshot=%v", res.Stack, res.SnapshotID, res.NoSnapshot)
	}
}

// TestR87_RotationCoversEveryAppInAsManyNights — Scenario D end to end.
func TestR87_RotationCoversEveryAppInAsManyNights(t *testing.T) {
	stacks := []string{"bookstack", "docmost", "kimai", "opengist"}
	h := newProofHarness(t, stacks...)
	seen := map[string]bool{}
	for i := 0; i < len(stacks); i++ {
		res := h.m.ProveOffboxUnit(context.Background())
		if res.Stack == "" {
			t.Fatalf("night %d picked nothing; %d apps covered so far", i+1, len(seen))
		}
		if seen[res.Stack] {
			t.Fatalf("night %d re-picked %s — the rotation must move on", i+1, res.Stack)
		}
		seen[res.Stack] = true
		h.m.RecordProofVerdict(res)
	}
	if len(seen) != len(stacks) {
		t.Fatalf("four nights must cover four apps; covered %v", seen)
	}
	// A fifth night has nothing due.
	if res := h.m.ProveOffboxUnit(context.Background()); !res.NoSnapshot {
		t.Fatalf("a fifth night must find nothing due; got %q", res.Stack)
	}
}

// ── B7, B8 ───────────────────────────────────────────────────────────────────────────────────────

// TestR87_HeadroomRefusalHappensBeforeTheDownload — a gate after the download is not a gate.
func TestR87_HeadroomRefusalHappensBeforeTheDownload(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.m.SetOffboxFreeFn(func(string) int64 { return 1 << 20 }) // 1 MiB, well under the floor

	res := h.m.ProveOffboxUnit(context.Background())
	if res.Err == nil {
		t.Fatal("an unwritable-headroom box must refuse")
	}
	for _, args := range h.allArgs() {
		if containsArg(args, "restore") {
			t.Fatalf("the refusal must happen BEFORE any download; a restore was issued: %v", args)
		}
	}
	// The customer-grade wording is shared with the restore path (REUSE.md's offsiteNoSpaceMsgFmt
	// rule). ASCII-only fragment, because an accented grep has returned 0 for strings that were there.
	if !strings.Contains(res.Err.Error(), "szabad hely") {
		t.Fatalf("the refusal must use the shared headroom wording; got %q", res.Err.Error())
	}
	if res.Verdict() != "" {
		t.Fatal("a refusal reaches no verdict about the backup")
	}
}

func TestR87_NoSnapshotYetIsNotAFailure(t *testing.T) {
	h := newProofHarness(t, "newapp")
	h.snapFor["newapp"] = "" // never backed up
	res := h.m.ProveOffboxUnit(context.Background())
	if !res.NoSnapshot {
		t.Fatalf("an app with no off-site snapshot is not a failure of this test; got %+v", res)
	}
	if res.Err != nil || res.Verdict() != "" {
		t.Fatalf("it must not be an error and must reach no verdict; err=%v verdict=%q", res.Err, res.Verdict())
	}
}

// TestR87_RestoreFailureIsNotAVerdictAboutTheBackup — a download that did not finish has seen
// nothing, so it must never become the "intact but empty" alarm.
func TestR87_RestoreFailureIsNotAVerdictAboutTheBackup(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.failRestore = true
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Err == nil {
		t.Fatal("a failed restore must surface as an error")
	}
	if res.Verdict() != "" {
		t.Fatalf("a failed restore must reach NO verdict; got %q", res.Verdict())
	}
	h.m.RecordProofVerdict(res)
	if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "" || len(tt.ProvedSnapshots) != 0 {
		t.Fatalf("a failed restore must not advance due-ness; got %+v", tt)
	}
}

// TestR87_HollowUnitReachesAFailVerdictThroughTheWholeJob — Scenario B through the job, not just the
// predicate. The judgement is right in isolation and could still be wired to nothing.
func TestR87_HollowUnitReachesAFailVerdictThroughTheWholeJob(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.unitFor["kimai"] = unitFixture{compose: composeWithDB} // the R-403 shape
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Verdict() != string(UnitProofFail) || res.Judgement.Reason != ProofReasonNoDatabaseDump {
		t.Fatalf("the job must reach a FAIL verdict on a hollow unit; got %q/%q", res.Verdict(), res.Judgement.Reason)
	}
	h.m.RecordProofVerdict(res)
	if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "fail" || tt.LastProofReason != string(ProofReasonNoDatabaseDump) {
		t.Fatalf("the failing verdict must be persisted with its reason; got result=%q reason=%q", tt.LastProofResult, tt.LastProofReason)
	}
}

// TestR87_CannotJudgeIsRecordedAndNotAPass.
func TestR87_CannotJudgeIsRecordedAndNotAPass(t *testing.T) {
	h := newProofHarness(t, "kimai")
	h.unitFor["kimai"] = unitFixture{dbDumps: []string{"d.sql"}} // no compose materialised
	res := h.m.ProveOffboxUnit(context.Background())
	if res.Verdict() != string(UnitProofCannotJudge) {
		t.Fatalf("a unit with no compose must be CANNOT JUDGE; got %q/%q", res.Verdict(), res.Judgement.Reason)
	}
	h.m.RecordProofVerdict(res)
	if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "cannot_judge" {
		t.Fatalf("cannot-judge must be recorded as itself; got %q", tt.LastProofResult)
	}
}

// TestR87_VerdictIsPublishedOnTheReportStatus — Part 2.4, including the absence rule.
func TestR87_VerdictIsPublishedOnTheReportStatus(t *testing.T) {
	h := newProofHarness(t, "kimai")

	// Before any proof, the field must be ABSENT — not "fail", not "pass".
	if st := h.m.OffboxReportStatus(); st == nil || st.LastProofResult != "" {
		t.Fatalf("a box that has never proved must report NOT RECORDED; got %+v", st)
	}
	res := h.m.ProveOffboxUnit(context.Background())
	h.m.RecordProofVerdict(res)

	st := h.m.OffboxReportStatus()
	if st.LastProofResult != "pass" || st.LastProofStack != "kimai" || st.LastProofSnapshot == "" || st.LastProofRun == "" {
		t.Fatalf("the verdict must reach the report status; got %+v", st)
	}
	// omitempty must keep the absent case off the wire entirely, or a reader cannot tell it apart.
	b, err := json.Marshal(&OffboxReportStatus{})
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(b), "last_proof_result") {
		t.Fatalf("an empty status must not emit last_proof_result at all; got %s", b)
	}
}