felhom-controller/controller/internal/backup/r691_kept_offsite_test.go

package backup

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

// R-691 (2) — „Use my kept data" / Load also look at the OFF-SITE copy (controller v0.277.0).
//
// Driven through the real KeptBestCopy / LoadKeptOffsite with the two restic seams (SetOffboxRunner answers
// `snapshots --json` and materialises the unit a `restore --include` asks for) and the unit-restore seam
// (SetKeptUnitRestoreFn records the call instead of reaching Docker). No restic, no ssh, no docker.

const r691App = "nextcloud"

type r691Harness struct {
	*proofHarness
	snapAt    time.Time
	snapPaths []string
	failSnaps bool
	// manifest is what the downloaded unit's manifest.json holds (raw JSON).
	manifest string
	unitHDD  string
	// restored records every unit restore call (unitDir); prepared counts prepare calls.
	restored []string
	prepared int
	done     chan bool
}

func newR691Harness(t *testing.T) *r691Harness {
	t.Helper()
	ph := newProofHarness(t) // nothing deployed: nextcloud is a REMOVED app with kept files
	h := &r691Harness{proofHarness: ph, snapAt: time.Now().Add(-time.Hour), done: make(chan bool, 1)}
	h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/backups/primary/" + r691App, "/mnt/felhom-drives/hdd_1/appdata/" + r691App}
	h.unitHDD = ph.drive
	h.manifest = r691Manifest(true)
	ph.m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		ph.mu.Lock()
		ph.argv = append(ph.argv, append([]string{}, args...))
		ph.mu.Unlock()
		switch {
		case containsArg(args, "snapshots"):
			if h.failSnaps {
				return []byte("ssh: connect refused"), errors.New("exit status 1")
			}
			b, _ := json.Marshal([]map[string]interface{}{{
				"short_id": "ab12cd34", "id": "ab12cd34ffff", "time": h.snapAt, "tags": []string{r691App}, "paths": h.snapPaths}})
			return b, nil
		case containsArg(args, "restore"):
			if ph.failRestore {
				return []byte("simulated restic failure"), os.ErrPermission
			}
			target, include := argValue(args, "--target"), argValue(args, "--include")
			dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
			for _, sub := range []string{"compose", "db-dumps"} {
				_ = os.MkdirAll(filepath.Join(dest, sub), 0o755)
			}
			_ = os.WriteFile(filepath.Join(dest, "db-dumps", "nextcloud-mariadb.sql"), []byte("x"), 0o644)
			_ = os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte("services:\n  nextcloud:\n    image: nextcloud:31\n  db:\n    image: mariadb:11.8\n"), 0o644)
			_ = os.WriteFile(filepath.Join(dest, "compose", "app.yaml"), []byte("env:\n  HDD_PATH: "+h.unitHDD+"\n"), 0o600)
			_ = os.WriteFile(filepath.Join(dest, "manifest.json"), []byte(h.manifest), 0o644)
			return nil, nil
		}
		return nil, nil
	})
	ph.m.SetKeptUnitRestoreFn(func(app, unitDir string) (UnitRestoreResult, error) {
		// The unit must still be on disk while the restore reads it.
		if _, err := os.Stat(UnitManifestFile(unitDir)); err != nil {
			t.Errorf("the unit restore ran on %s, which no longer holds a manifest: %v", unitDir, err)
		}
		h.restored = append(h.restored, unitDir)
		return UnitRestoreResult{ManifestDBs: 1, DBsReplayed: 1}, nil
	})
	return h
}

// r691Manifest is a unit manifest listing one dump; withData adds the `data` block (the data's version).
func r691Manifest(withData bool) string {
	man := map[string]interface{}{"schema_version": 2, "app_name": r691App, "db_dumps": []string{"nextcloud-mariadb.sql"}}
	if withData {
		man["data"] = map[string]interface{}{"at": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339),
			"image_pins": []string{"mariadb:11.8", "nextcloud:31"}}
	}
	b, _ := json.Marshal(man)
	return string(b)
}

func (h *r691Harness) load(t *testing.T, c KeptCopy) (ok bool) {
	t.Helper()
	h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
		func(error) string { return "ok" }, func(err error) string { return err.Error() },
		func(ok bool) { h.done <- ok })
	select {
	case ok = <-h.done:
	case <-time.After(10 * time.Second):
		t.Fatal("the load never finished")
	}
	return ok
}

// The consequence, not the mechanism: which copy the household is offered.
// COMPANION RED-PROOF: KeptBestCopy returning KeptDBCopy alone (0.276.0's choice) → the first assertion fails
// with ok=false (an app whose only database copy is off-site gets „no backup").
func TestR691_OffsiteCopyIsOfferedWhenItIsTheOnlyOrNewest(t *testing.T) {
	h := newR691Harness(t)
	c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
	if !ok || c.Tier != KeptTierOffsite || c.SnapshotID != "ab12cd34" || c.UnitPath != h.snapPaths[0] {
		t.Fatalf("only off-site copy: got %+v ok=%v, want the off-site snapshot ab12cd34", c, ok)
	}
	if !c.Time.Equal(h.snapAt) {
		t.Fatalf("the copy is dated %v, want the snapshot's time %v", c.Time, h.snapAt)
	}

	// A local unit NEWER than the snapshot wins — no download.
	unit := RecoveryUnitPath(h.m.namespaceRoot(h.drive), r691App)
	writeKeptUnit(t, unit, h.drive, true, time.Now())
	h.prov.hdd[r691App] = "" // removed
	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != 1 || c.UnitDir != unit {
		t.Fatalf("a newer local unit must win: got %+v ok=%v", c, ok)
	}
	// An OLDER local unit loses to the off-site copy.
	old := time.Now().Add(-3 * time.Hour)
	writeKeptUnit(t, unit, h.drive, true, old)
	for _, f := range []string{UnitManifestFile(unit), filepath.Join(unit, "compose", "app.yaml")} {
		_ = os.Chtimes(f, old, old)
	}
	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != KeptTierOffsite {
		t.Fatalf("an older local unit must lose to the off-site copy: got %+v ok=%v", c, ok)
	}

	// Negative controls: a snapshot with no unit, an unreadable repository, an INSTALLED app.
	_ = os.RemoveAll(unit)
	h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/appdata/" + r691App}
	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
		t.Fatalf("a snapshot holding no recovery unit was offered: %+v", c)
	}
	h.snapPaths = []string{"/x/backups/primary/" + r691App}
	h.failSnaps = true
	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
		t.Fatalf("an unreadable repository offered a copy: %+v", c)
	}
	h.failSnaps = false
	h.prov.deployed[r691App] = true
	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
		t.Fatalf("an installed app's off-site copy was offered as kept data: %+v", c)
	}
}

// The load: download the unit ALONE, judge it, THEN prepare, THEN the one unit restore; the scratch goes.
// COMPANION RED-PROOF: drop the `man.Data == nil` refusal in downloadKeptOffsiteUnit → the no-data-block case
// reaches the restore (restored=1, prepared=1) and fails.
func TestR691_OffsiteLoadDownloadsJudgesThenRestores(t *testing.T) {
	h := newR691Harness(t)
	c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
	if !ok {
		t.Fatal("no off-site copy offered")
	}
	if !h.load(t, c) {
		t.Fatalf("a good off-site unit did not load: %+v", h.m.RestoreStatus())
	}
	if h.prepared != 1 || len(h.restored) != 1 {
		t.Fatalf("prepare=%d restore=%d, want 1 and 1", h.prepared, len(h.restored))
	}
	scratch := h.proofScratch(t, r691App)
	if !strings.HasPrefix(h.restored[0], scratch+string(filepath.Separator)) || !strings.HasSuffix(h.restored[0], "/backups/primary/"+r691App) {
		t.Fatalf("the restore read %s, want the downloaded unit inside %s", h.restored[0], scratch)
	}
	var sawUnitOnly bool
	for _, a := range h.allArgs() {
		if containsArg(a, "restore") && argValue(a, "--include") == h.snapPaths[0] && containsArg(a, "ab12cd34") {
			sawUnitOnly = true
		}
	}
	if !sawUnitOnly {
		t.Fatalf("no unit-only restore of snapshot ab12cd34 (--include %s): %v", h.snapPaths[0], h.allArgs())
	}
	if _, err := os.Stat(scratch); !os.IsNotExist(err) {
		t.Fatalf("the downloaded copy was left behind at %s (err=%v)", scratch, err)
	}
}

// Every refusal happens BEFORE prepare: the kept files are exactly as they were, nothing is restored.
func TestR691_OffsiteLoadRefusalsLeaveTheKeptFilesAlone(t *testing.T) {
	cases := []struct {
		name string
		set  func(h *r691Harness)
		want error
	}{
		{"data version not recorded", func(h *r691Harness) { h.manifest = r691Manifest(false) }, ErrKeptOffsiteVersionUnknown},
		{"taken of another drive", func(h *r691Harness) { h.unitHDD = "/mnt/felhom-drives/other" }, ErrKeptOffsiteNotUsable},
		{"definition is not the data's", func(h *r691Harness) {
			h.manifest = strings.Replace(r691Manifest(true), "nextcloud:31", "nextcloud:30", 1)
		}, ErrUnitVersionMismatch},
		{"download failed", func(h *r691Harness) { h.failRestore = true }, nil},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			h := newR691Harness(t)
			kept := filepath.Join(h.drive, "appdata", r691App, "photo.jpg")
			_ = os.MkdirAll(filepath.Dir(kept), 0o755)
			_ = os.WriteFile(kept, []byte("the household's photo"), 0o644)
			c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
			if !ok {
				t.Fatal("no off-site copy offered")
			}
			tc.set(h)
			var got error
			h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
				func(error) string { return "ok" }, func(err error) string { got = err; return err.Error() },
				func(ok bool) { h.done <- ok })
			if <-h.done {
				t.Fatal("the load reported success")
			}
			if h.prepared != 0 || len(h.restored) != 0 {
				t.Fatalf("prepare=%d restore=%d after a refusal, want 0 and 0", h.prepared, len(h.restored))
			}
			if tc.want != nil && !errors.Is(got, tc.want) {
				t.Fatalf("refused with %v, want %v", got, tc.want)
			}
			if b, err := os.ReadFile(kept); err != nil || string(b) != "the household's photo" {
				t.Fatalf("the kept file changed: %q %v", b, err)
			}
			if st := h.m.RestoreStatus(); st.Running || st.Last == nil || st.Last.OK {
				t.Fatalf("the restore record does not read as failed: %+v", st)
			}
			if _, err := os.Stat(h.proofScratch(t, r691App)); !os.IsNotExist(err) {
				t.Fatalf("the downloaded copy was left behind after a refusal (err=%v)", err)
			}
		})
	}
}