felhom-controller/controller/internal/backup/r414_reachability_test.go

package backup

import (
	"context"
	"encoding/json"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// R-411 A2 + R-414 — the collision that can no longer happen, and the box that can now be proved.

// TestR411_IntegrityCheckSkipsWhileARestoreHoldsIt — A2, the whole point of Part 1.
//
// This is last night's collision, from the other side: with the restore now holding the flag, the
// integrity check must SKIP rather than run, meet the lock and delete it.
func TestR411_IntegrityCheckSkipsWhileARestoreHoldsIt(t *testing.T) {
	h := newLockHarness(t, "kimai")

	// Stand in for a restore in flight: it now takes the flag, so hold it.
	if err := h.m.AcquireRunningForTest(); err != nil {
		t.Fatal(err)
	}
	before := h.m.settings.GetOffboxTarget().LastIntegrityCheck

	res := h.m.CheckOffboxIntegrity(context.Background())

	if !res.Skipped {
		t.Fatalf("the check must SKIP while a restore holds the flag; got %+v", res)
	}
	if len(h.allArgs()) != 0 {
		t.Fatalf("a skipped check must invoke restic ZERO times — this is the non-effect R-411 is about; got %v", h.allArgs())
	}
	for _, args := range h.allArgs() {
		if containsArg(args, "unlock") || containsArg(args, "--remove-all") {
			t.Fatalf("the unlock escalation fired: %v", args)
		}
	}
	if got := h.m.settings.GetOffboxTarget().LastIntegrityCheck; got != before {
		t.Fatalf("a skip must NOT advance due-ness; %q -> %q", before, got)
	}
}

// ── R-414 ───────────────────────────────────────────────────────────────────────────────────────

// drivelessHarness is `demo-felhom`'s real shape: deployed apps, and ZERO registered storage paths.
func drivelessHarness(t *testing.T, stacks ...string) (*Manager, *settings.Settings, *[][]string) {
	t.Helper()
	m, sett := newOffboxManager(t)
	// deliberately NO AddStoragePath — that is the whole point
	deployed := map[string]bool{}
	for _, s := range stacks {
		deployed[s] = true
	}
	m.SetStackProvider(&offbox3aProvider{
		hdd: map[string]string{}, binds: map[string][]ClassifiedBind{},
		has: map[string]bool{}, deployed: deployed,
	})
	var argv [][]string
	m.SetOffboxLatestSnapshotFn(func(_ context.Context, stack string) (string, []string, error) {
		return "snap-" + stack, []string{"/mnt/sys_drive/felhom-data/backups/primary/" + stack}, nil
	})
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		argv = append(argv, append([]string{}, args...))
		if containsArg(args, "restore") {
			target, include := argValue(args, "--target"), argValue(args, "--include")
			if target != "" && include != "" {
				dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
				materialiseUnit(t, dest, unitFixture{})
			}
		}
		return []byte("{}"), nil
	})
	m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
	return m, sett, &argv
}

// TestR414_DrivelessBoxReachesAVerdict — C1.
//
// RED-PROOF (run 2026-09-01): reverting the resolver to refuse (dropping step 4) AND restoring the
// Err path makes this fail with `last_proof_result` absent — which is the exact state `demo-felhom`
// was in every night.
func TestR414_DrivelessBoxReachesAVerdict(t *testing.T) {
	m, sett, _ := drivelessHarness(t, "opengist")

	res := m.ProveOffboxUnit(context.Background())
	if res.Verdict() == "" {
		t.Fatalf("a driveless box must reach a RECORDED verdict, never an empty one — empty is what the hub reads as 'controller too old'; got %+v", res)
	}
	m.RecordProofVerdict(res)

	got := sett.GetOffboxTarget().LastProofResult
	if got == "" {
		t.Fatal("last_proof_result is ABSENT after a run on a driveless box — this is R-414 exactly: the hub cannot tell 'cannot run here' from 'too old'")
	}
	// With the unit-only fallback in place the box can actually be proved.
	if got != string(UnitProofPass) && got != ProofResultCannotRun {
		t.Fatalf("unexpected verdict %q — expected a real judgement (the fallback worked) or %q", got, ProofResultCannotRun)
	}
	t.Logf("driveless box reached verdict %q", got)
}

// TestR414_UnitOnlyRestoreFallsBackToSystemData — C3.
func TestR414_UnitOnlyRestoreFallsBackToSystemData(t *testing.T) {
	m, _, _ := drivelessHarness(t, "opengist")
	scratch, nsRoot, err := m.offboxProofScratchDir("opengist")
	if err != nil {
		t.Fatalf("a UNIT-ONLY scratch must resolve on a driveless box (R-414); got %v", err)
	}
	sys := m.cfg.Paths.SystemDataPath
	if !strings.HasPrefix(filepath.Clean(scratch), filepath.Clean(sys)) {
		t.Fatalf("the unit-only scratch must fall back to the system data path %q; got %q", sys, scratch)
	}
	if nsRoot == "" {
		t.Fatal("the namespace root must be returned for the free-space probe")
	}
	if !strings.Contains(scratch, "offsite-proof") {
		t.Fatalf("it must still land in the PROOF root, not the customer's; got %q", scratch)
	}
}

// TestR414_FullRestoreDoesNotFallBack — C4. The state-only tier is protected.
func TestR414_FullRestoreDoesNotFallBack(t *testing.T) {
	m, _, _ := drivelessHarness(t, "opengist")
	_, _, err := m.offboxRestoreScratchDir("opengist")
	if err == nil {
		t.Fatal("the CUSTOMER's scratch must still refuse on a driveless box — a full restore pulls bulk userdata and the internal SSD is a state-only tier (07 §2.2)")
	}
	// ASCII fragment, with a negative control, because an accented grep has returned 0 for strings
	// that were there (R-364).
	if !strings.Contains(err.Error(), "adatmeghajt") {
		t.Fatalf("the R-252 refusal wording must be preserved — it tells the customer what to do; got %q", err.Error())
	}
	if strings.Contains(err.Error(), "ZZZ-NOT-IN-THE-MESSAGE") {
		t.Fatal("negative control matched — the fragment search is not discriminating")
	}
}

// TestR414_AbsentStillMeansNotRecorded — C2. The two meanings must stay distinct.
func TestR414_AbsentStillMeansNotRecorded(t *testing.T) {
	m, sett, _ := drivelessHarness(t, "opengist")
	if got := sett.GetOffboxTarget().LastProofResult; got != "" {
		t.Fatalf("a box that has never proved must report ABSENT; got %q", got)
	}
	// A skip must still record nothing — only a reached outcome writes.
	if err := m.AcquireRunningForTest(); err != nil {
		t.Fatal(err)
	}
	res := m.ProveOffboxUnit(context.Background())
	m.RecordProofVerdict(res)
	if got := sett.GetOffboxTarget().LastProofResult; got != "" {
		t.Fatalf("a SKIP must leave the field absent — it looked at nothing; got %q", got)
	}
	m.ReleaseRunningForTest()

	// And the wire must not carry the key at all when absent.
	b, err := json.Marshal(&OffboxReportStatus{})
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(b), "last_proof_result") {
		t.Fatalf("an empty status must not emit last_proof_result; got %s", b)
	}
}

// TestR414_CannotRunIsRecordedButDoesNotAdvanceDueness — the half that keeps the app retryable.
func TestR414_CannotRunIsRecordedButDoesNotAdvanceDueness(t *testing.T) {
	m, sett, _ := drivelessHarness(t, "opengist")
	res := ProofResult{Stack: "opengist", SnapshotID: "snap-opengist",
		CannotRun: true, CannotWhy: "nincs regisztralt adatmeghajto"}
	if res.Verdict() != ProofResultCannotRun {
		t.Fatalf("cannot-run must render as %q, never as empty; got %q", ProofResultCannotRun, res.Verdict())
	}
	m.RecordProofVerdict(res)
	tt := sett.GetOffboxTarget()
	if tt.LastProofResult != ProofResultCannotRun {
		t.Fatalf("cannot-run must be RECORDED so the hub can see it; got %q", tt.LastProofResult)
	}
	if len(tt.ProvedSnapshots) != 0 {
		t.Fatalf("cannot-run must NOT advance per-snapshot due-ness — nothing was proved, and marking it proved would stop the app ever being retried; got %v", tt.ProvedSnapshots)
	}
	if tt.LastProofSnapshot != "" {
		t.Fatalf("cannot-run must not claim a proved snapshot; got %q", tt.LastProofSnapshot)
	}
}

// TestR414_NoCustomerAlarm — C5. The customer's backups are fine and there is nothing for them to do.
func TestR414_NoCustomerAlarm(t *testing.T) {
	m, _, _ := drivelessHarness(t, "opengist")
	var pushed int
	m.SetOffboxOrphanEvent(func(string, string) { pushed++ })
	res := m.ProveOffboxUnit(context.Background())
	m.RecordProofVerdict(res)
	if pushed != 0 {
		t.Fatalf("a driveless box must raise no event from the backup layer; got %d", pushed)
	}
}