felhom-controller/controller/internal/backup/r358_scratch_marker_test.go

package backup

import (
	"bytes"
	"context"
	"encoding/json"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// ── R-358 — a failed download was offered as a good one ──────────────────────────────────────────
//
// `OffboxFullScratchReady` used to answer "the directory exists and is non-empty". A restic run that
// dies part-way leaves exactly that. So the product showed „Teljes visszaállítás indítása" over a
// part-copy and pressing it reported success — observed on demo-hp 2026-08-21.
//
// A non-empty directory is evidence that SOMETHING was written, never that everything was. The marker
// carries the only fact that distinguishes them: did the run FINISH, and was it the full one.

// newR358Manager gives a manager whose scratch resolves into a t.TempDir().
func newR358Manager(t *testing.T) (*Manager, string) {
	t.Helper()
	m, sett := newOffboxManager(t)
	drive := t.TempDir()
	if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "drive", Schedulable: true}); err != nil {
		t.Fatal(err)
	}
	m.SetStackProvider(&offbox3aProvider{
		hdd: map[string]string{"kimai": drive}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{},
	})
	scratch, _, err := m.offboxRestoreScratchDir("kimai")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.MkdirAll(scratch, 0o755); err != nil {
		t.Fatal(err)
	}
	return m, scratch
}

// writeScratchPayload plants the files a part-way restic run leaves behind.
func writeScratchPayload(t *testing.T, scratch string) {
	t.Helper()
	if err := os.MkdirAll(filepath.Join(scratch, "backups", "primary", "kimai"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(scratch, "backups", "primary", "kimai", "half.tar"), []byte("partial"), 0o644); err != nil {
		t.Fatal(err)
	}
}

func TestR358_FailedRestoreLeavesNoUsableScratch(t *testing.T) {
	m, scratch := newR358Manager(t)
	writeScratchPayload(t, scratch) // files present, run never finished → no marker

	if m.OffboxFullScratchReady("kimai") {
		t.Fatal("a part-copy was reported READY — this is the defect: the customer is offered " +
			"„Teljes visszaállítás indítása" + " over a download that never finished")
	}
}

func TestR358_UnitOnlyScratchIsNotFullReady(t *testing.T) {
	m, scratch := newR358Manager(t)
	writeScratchPayload(t, scratch)
	if err := m.writeScratchMarker(scratch, "snap-1", false); err != nil {
		t.Fatal(err)
	}
	if m.OffboxFullScratchReady("kimai") {
		t.Fatal("a UNIT-ONLY scratch was reported ready for a full restore — both modes write the " +
			"same directory, so only the marker's `full` field separates them")
	}
}

func TestR358_StaleMarkerIsClearedBeforeTheRun(t *testing.T) {
	m, scratch := newR358Manager(t)
	if err := m.writeScratchMarker(scratch, "snap-OLD", true); err != nil {
		t.Fatal(err)
	}
	if !m.OffboxFullScratchReady("kimai") {
		t.Fatal("fixture wrong: a valid full marker should read ready")
	}
	// What the next run does before restic touches anything.
	m.clearScratchMarker(scratch)
	writeScratchPayload(t, scratch) // ...and then that run dies part-way

	if m.OffboxFullScratchReady("kimai") {
		t.Fatal("a marker from a PREVIOUS run certified a later part-copy — the stale certificate " +
			"is the whole reason the clear happens before restic, not after")
	}
}

func TestR358_UnreadableMarkerFailsClosed(t *testing.T) {
	var buf bytes.Buffer
	m, scratch := newR358Manager(t)
	m.logger = log.New(&buf, "", 0)
	writeScratchPayload(t, scratch)
	if err := os.WriteFile(filepath.Join(scratch, scratchMarkerName), []byte("{not json"), 0o600); err != nil {
		t.Fatal(err)
	}

	if m.OffboxFullScratchReady("kimai") {
		t.Fatal("an unparseable marker was treated as a completion certificate")
	}
	if !strings.Contains(buf.String(), "WARN") {
		t.Errorf("a scratch refused for an unreadable marker must say so — silence makes a refusal "+
			"indistinguishable from a missing download; log was %q", buf.String())
	}
}

func TestR358_WrongSchemaFailsClosed(t *testing.T) {
	m, scratch := newR358Manager(t)
	writeScratchPayload(t, scratch)
	if err := os.WriteFile(filepath.Join(scratch, scratchMarkerName),
		[]byte(`{"schema":99,"snapshot_id":"s","full":true,"finished_at":"2026-08-30T00:00:00Z"}`), 0o600); err != nil {
		t.Fatal(err)
	}
	if m.OffboxFullScratchReady("kimai") {
		t.Fatal("a marker with an unrecognised schema was accepted — a format we cannot read is not a certificate")
	}
}

func TestR358_CompletedFullScratchStillReady(t *testing.T) {
	// The happy path is unchanged: a finished full download is still offered.
	m, scratch := newR358Manager(t)
	writeScratchPayload(t, scratch)
	if err := m.writeScratchMarker(scratch, "snap-1", true); err != nil {
		t.Fatal(err)
	}
	if !m.OffboxFullScratchReady("kimai") {
		t.Fatal("a COMPLETED full restore is no longer offered — the fix broke the thing it protects")
	}
}

func TestR358_MarkerIsWrittenAt0600AndAtomically(t *testing.T) {
	m, scratch := newR358Manager(t)
	if err := m.writeScratchMarker(scratch, "snap-1", true); err != nil {
		t.Fatal(err)
	}
	fi, err := os.Stat(filepath.Join(scratch, scratchMarkerName))
	if err != nil {
		t.Fatal(err)
	}
	if fi.Mode().Perm() != 0o600 {
		t.Errorf("marker mode = %v, want 0600", fi.Mode().Perm())
	}
	// The tmp file must not survive: a leftover .tmp beside the marker is a torn write that a later
	// reader could mistake for the real thing.
	if _, err := os.Stat(filepath.Join(scratch, scratchMarkerName+".tmp")); !os.IsNotExist(err) {
		t.Error("the temporary marker file was left behind")
	}
}

// TestR358_MarkerIsNeverPlaced pins the assumption the whole design rests on: placement is driven by
// the SNAPSHOT's own path list, not by a directory walk, so a file that exists only locally cannot be
// copied into the customer's live data. Stated as a test rather than trusted as a comment — the spec
// asked for exactly this, and "a comment asserting an invariant needs a test pinning it" is a standing
// rule earned nine times over in this project.
func TestR358_MarkerIsNeverPlaced(t *testing.T) {
	const stack = "kimai"
	oldNs := "/mnt/old"
	scratch := t.TempDir()
	liveNs := t.TempDir()
	snapPaths := []string{
		oldNs + "/backups/primary/" + stack,
		oldNs + "/appdata/" + stack,
	}

	placements, err := mapOffsiteRestorePaths(snapPaths, stack, scratch, liveNs)
	if err != nil {
		t.Fatalf("mapOffsiteRestorePaths: %v", err)
	}
	if len(placements) == 0 {
		t.Fatal("fixture produced no placements — the test would prove nothing")
	}
	for _, pl := range placements {
		if strings.Contains(pl.src, scratchMarkerName) || strings.Contains(pl.dst, scratchMarkerName) {
			t.Fatalf("the completion marker entered a placement (src=%q dst=%q) — it would be copied "+
				"into the customer's live data", pl.src, pl.dst)
		}
	}
}

// TestR358_MarkerOrderingIsExecuted — R-398, and the row that prompted it was WRONG in a way worth
// recording rather than quietly fixing.
//
// This test used to walk the AST of RestoreOffboxScratch, on the stated ground that "`resticStep` is
// not a seam, so the ORDER cannot be proven by execution". **The first half is true and the conclusion
// was false.** `resticStep` is not overridable, but the layer it calls — `offboxRunner`, injected by
// `SetOffboxRunner` — has been a seam since the off-site tier shipped, and other tests in this package
// have been driving restic-backed paths through it all along. R-398 was filed off my own mistaken
// reading; it is corrected, not closed.
//
// Executing it is strictly stronger than the AST walk, and not only because it runs the real function:
// the runner sees EVERY argv, including the `unlock --remove-all` escalation that `resticStep` performs
// on a lock error. A `resticStepFn` seam — which R-398 proposed and this test's existence argued for —
// would have REPLACED that escalation and hidden it from exactly the assertions that need to see it.
//
// The order is the whole safety property: a marker written before restic certifies a download that has
// not happened, and a clear that runs after it leaves a stale certificate over a fresh part-copy.
func TestR358_MarkerOrderingIsExecuted(t *testing.T) {
	m, scratch := newR358Manager(t)
	// A stale marker from a previous run, so "cleared before" is observable as a state change rather
	// than as an absence that was always absent.
	if err := m.writeScratchMarker(scratch, "snap-OLD", true); err != nil {
		t.Fatal(err)
	}

	var events []string
	markerAtRestic := "unset"
	m.SetOffboxLatestSnapshotFn(func(context.Context, string) (string, []string, error) {
		return "snap-NEW", []string{"/mnt/old/backups/primary/kimai"}, nil
	})
	m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case containsArg(args, "cat") && containsArg(args, "config"):
			return []byte(`{"version":2}`), nil
		case containsArg(args, "stats"):
			return []byte(`{"total_size":1024}`), nil
		case containsArg(args, "restore"):
			events = append(events, "restic-restore")
			// THE OBSERVATION: what does the marker look like at the moment restic runs? If the clear
			// happened first it is gone; if the write happened first it is already there, certifying a
			// download that has not finished.
			if _, err := os.Stat(filepath.Join(scratch, scratchMarkerName)); err == nil {
				markerAtRestic = "present"
			} else {
				markerAtRestic = "absent"
			}
			return nil, nil
		case containsArg(args, "unlock"):
			// TWO DIFFERENT UNLOCKS, and only one of them is dangerous. Plain `unlock` is restic's
			// stale-ONLY sweep, run as pre-run hygiene by unlockStale before every restore.
			// `unlock --remove-all` is resticStep's crash-lock escalation, and it is the act that must
			// never happen beside a live sibling operation. Recording them separately here is what
			// lets the R-359 lock-safety tests assert the second is absent without tripping over the
			// first — a distinction the AST walk this replaced could not have surfaced at all.
			if containsArg(args, "--remove-all") {
				events = append(events, "unlock--remove-all")
			} else {
				events = append(events, "unlock-stale")
			}
			return nil, nil
		}
		return nil, nil
	})

	if err := m.RestoreOffboxScratch(context.Background(), "kimai", true); err != nil {
		t.Fatalf("RestoreOffboxScratch: %v", err)
	}

	if markerAtRestic != "absent" {
		t.Fatalf("the completion marker was %s while restic was still downloading — a marker that "+
			"predates the download certifies a copy that does not exist yet (and here it was the "+
			"PREVIOUS run's marker, covering a fresh part-copy)", markerAtRestic)
	}
	// The restore must actually have run, or the ordering assertion above proves nothing. It is NOT
	// asserted to be first: `unlockStale` legitimately precedes it as pre-run hygiene, which this
	// execution test surfaced on its first run and the AST walk could never have shown.
	if !containsEvent(events, "restic-restore") {
		t.Fatalf("restic never ran, so this test proves nothing about ordering (events=%v)", events)
	}
	// And the DANGEROUS unlock never appeared: nothing here met a lock, so resticStep never escalated.
	if containsEvent(events, "unlock--remove-all") {
		t.Fatalf("`unlock --remove-all` ran during an ordinary restore (events=%v) — that escalation "+
			"is only safe because the single-writer flag proves no sibling is live", events)
	}
	// And after a successful run the marker is there, carrying THIS run's snapshot, not the old one.
	data, err := os.ReadFile(filepath.Join(scratch, scratchMarkerName))
	if err != nil {
		t.Fatalf("no marker after a successful restore: %v", err)
	}
	var mk scratchMarker
	if err := json.Unmarshal(data, &mk); err != nil {
		t.Fatal(err)
	}
	if mk.SnapshotID != "snap-NEW" || !mk.Full {
		t.Fatalf("marker = %+v, want snapshot snap-NEW and full=true — the stale one survived", mk)
	}
	if !m.OffboxFullScratchReady("kimai") {
		t.Fatal("a completed full restore did not read as ready")
	}
}

func containsEvent(ev []string, want string) bool {
	for _, e := range ev {
		if e == want {
			return true
		}
	}
	return false
}

// containsArg is a local helper so this file does not depend on another test file's ordering.
func containsArg(args []string, want string) bool {
	for _, a := range args {
		if a == want {
			return true
		}
	}
	return false
}