package backup import ( "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" ) // R-691 (2) — „Use my kept data" / Load also look at the OFF-SITE copy (controller v0.277.0). // // Driven through the real KeptBestCopy / LoadKeptOffsite with the two restic seams (SetOffboxRunner answers // `snapshots --json` and materialises the unit a `restore --include` asks for) and the unit-restore seam // (SetKeptUnitRestoreFn records the call instead of reaching Docker). No restic, no ssh, no docker. const r691App = "nextcloud" type r691Harness struct { *proofHarness snapAt time.Time snapPaths []string failSnaps bool // manifest is what the downloaded unit's manifest.json holds (raw JSON). manifest string unitHDD string // restored records every unit restore call (unitDir); prepared counts prepare calls. restored []string prepared int done chan bool } func newR691Harness(t *testing.T) *r691Harness { t.Helper() ph := newProofHarness(t) // nothing deployed: nextcloud is a REMOVED app with kept files h := &r691Harness{proofHarness: ph, snapAt: time.Now().Add(-time.Hour), done: make(chan bool, 1)} h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/backups/primary/" + r691App, "/mnt/felhom-drives/hdd_1/appdata/" + r691App} h.unitHDD = ph.drive h.manifest = r691Manifest(true) ph.m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) { ph.mu.Lock() ph.argv = append(ph.argv, append([]string{}, args...)) ph.mu.Unlock() switch { case containsArg(args, "snapshots"): if h.failSnaps { return []byte("ssh: connect refused"), errors.New("exit status 1") } b, _ := json.Marshal([]map[string]interface{}{{ "short_id": "ab12cd34", "id": "ab12cd34ffff", "time": h.snapAt, "tags": []string{r691App}, "paths": h.snapPaths}}) return b, nil case containsArg(args, "restore"): if ph.failRestore { return []byte("simulated restic failure"), os.ErrPermission } target, include := argValue(args, "--target"), argValue(args, "--include") dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator))) for _, sub := range []string{"compose", "db-dumps"} { _ = os.MkdirAll(filepath.Join(dest, sub), 0o755) } _ = os.WriteFile(filepath.Join(dest, "db-dumps", "nextcloud-mariadb.sql"), []byte("x"), 0o644) _ = os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte("services:\n nextcloud:\n image: nextcloud:31\n db:\n image: mariadb:11.8\n"), 0o644) _ = os.WriteFile(filepath.Join(dest, "compose", "app.yaml"), []byte("env:\n HDD_PATH: "+h.unitHDD+"\n"), 0o600) _ = os.WriteFile(filepath.Join(dest, "manifest.json"), []byte(h.manifest), 0o644) return nil, nil } return nil, nil }) ph.m.SetKeptUnitRestoreFn(func(app, unitDir string) (UnitRestoreResult, error) { // The unit must still be on disk while the restore reads it. if _, err := os.Stat(UnitManifestFile(unitDir)); err != nil { t.Errorf("the unit restore ran on %s, which no longer holds a manifest: %v", unitDir, err) } h.restored = append(h.restored, unitDir) return UnitRestoreResult{ManifestDBs: 1, DBsReplayed: 1}, nil }) return h } // r691Manifest is a unit manifest listing one dump; withData adds the `data` block (the data's version). func r691Manifest(withData bool) string { man := map[string]interface{}{"schema_version": 2, "app_name": r691App, "db_dumps": []string{"nextcloud-mariadb.sql"}} if withData { man["data"] = map[string]interface{}{"at": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339), "image_pins": []string{"mariadb:11.8", "nextcloud:31"}} } b, _ := json.Marshal(man) return string(b) } func (h *r691Harness) load(t *testing.T, c KeptCopy) (ok bool) { t.Helper() h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil }, func(error) string { return "ok" }, func(err error) string { return err.Error() }, func(ok bool) { h.done <- ok }) select { case ok = <-h.done: case <-time.After(10 * time.Second): t.Fatal("the load never finished") } return ok } // The consequence, not the mechanism: which copy the household is offered. // COMPANION RED-PROOF: KeptBestCopy returning KeptDBCopy alone (0.276.0's choice) → the first assertion fails // with ok=false (an app whose only database copy is off-site gets „no backup"). func TestR691_OffsiteCopyIsOfferedWhenItIsTheOnlyOrNewest(t *testing.T) { h := newR691Harness(t) c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive) if !ok || c.Tier != KeptTierOffsite || c.SnapshotID != "ab12cd34" || c.UnitPath != h.snapPaths[0] { t.Fatalf("only off-site copy: got %+v ok=%v, want the off-site snapshot ab12cd34", c, ok) } if !c.Time.Equal(h.snapAt) { t.Fatalf("the copy is dated %v, want the snapshot's time %v", c.Time, h.snapAt) } // A local unit NEWER than the snapshot wins — no download. unit := RecoveryUnitPath(h.m.namespaceRoot(h.drive), r691App) writeKeptUnit(t, unit, h.drive, true, time.Now()) h.prov.hdd[r691App] = "" // removed if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != 1 || c.UnitDir != unit { t.Fatalf("a newer local unit must win: got %+v ok=%v", c, ok) } // An OLDER local unit loses to the off-site copy. old := time.Now().Add(-3 * time.Hour) writeKeptUnit(t, unit, h.drive, true, old) for _, f := range []string{UnitManifestFile(unit), filepath.Join(unit, "compose", "app.yaml")} { _ = os.Chtimes(f, old, old) } if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != KeptTierOffsite { t.Fatalf("an older local unit must lose to the off-site copy: got %+v ok=%v", c, ok) } // Negative controls: a snapshot with no unit, an unreadable repository, an INSTALLED app. _ = os.RemoveAll(unit) h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/appdata/" + r691App} if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok { t.Fatalf("a snapshot holding no recovery unit was offered: %+v", c) } h.snapPaths = []string{"/x/backups/primary/" + r691App} h.failSnaps = true if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok { t.Fatalf("an unreadable repository offered a copy: %+v", c) } h.failSnaps = false h.prov.deployed[r691App] = true if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok { t.Fatalf("an installed app's off-site copy was offered as kept data: %+v", c) } } // The load: download the unit ALONE, judge it, THEN prepare, THEN the one unit restore; the scratch goes. // COMPANION RED-PROOF: drop the `man.Data == nil` refusal in downloadKeptOffsiteUnit → the no-data-block case // reaches the restore (restored=1, prepared=1) and fails. func TestR691_OffsiteLoadDownloadsJudgesThenRestores(t *testing.T) { h := newR691Harness(t) c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive) if !ok { t.Fatal("no off-site copy offered") } if !h.load(t, c) { t.Fatalf("a good off-site unit did not load: %+v", h.m.RestoreStatus()) } if h.prepared != 1 || len(h.restored) != 1 { t.Fatalf("prepare=%d restore=%d, want 1 and 1", h.prepared, len(h.restored)) } scratch := h.proofScratch(t, r691App) if !strings.HasPrefix(h.restored[0], scratch+string(filepath.Separator)) || !strings.HasSuffix(h.restored[0], "/backups/primary/"+r691App) { t.Fatalf("the restore read %s, want the downloaded unit inside %s", h.restored[0], scratch) } var sawUnitOnly bool for _, a := range h.allArgs() { if containsArg(a, "restore") && argValue(a, "--include") == h.snapPaths[0] && containsArg(a, "ab12cd34") { sawUnitOnly = true } } if !sawUnitOnly { t.Fatalf("no unit-only restore of snapshot ab12cd34 (--include %s): %v", h.snapPaths[0], h.allArgs()) } if _, err := os.Stat(scratch); !os.IsNotExist(err) { t.Fatalf("the downloaded copy was left behind at %s (err=%v)", scratch, err) } } // Every refusal happens BEFORE prepare: the kept files are exactly as they were, nothing is restored. func TestR691_OffsiteLoadRefusalsLeaveTheKeptFilesAlone(t *testing.T) { cases := []struct { name string set func(h *r691Harness) want error }{ {"data version not recorded", func(h *r691Harness) { h.manifest = r691Manifest(false) }, ErrKeptOffsiteVersionUnknown}, {"taken of another drive", func(h *r691Harness) { h.unitHDD = "/mnt/felhom-drives/other" }, ErrKeptOffsiteNotUsable}, {"definition is not the data's", func(h *r691Harness) { h.manifest = strings.Replace(r691Manifest(true), "nextcloud:31", "nextcloud:30", 1) }, ErrUnitVersionMismatch}, {"download failed", func(h *r691Harness) { h.failRestore = true }, nil}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { h := newR691Harness(t) kept := filepath.Join(h.drive, "appdata", r691App, "photo.jpg") _ = os.MkdirAll(filepath.Dir(kept), 0o755) _ = os.WriteFile(kept, []byte("the household's photo"), 0o644) c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive) if !ok { t.Fatal("no off-site copy offered") } tc.set(h) var got error h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil }, func(error) string { return "ok" }, func(err error) string { got = err; return err.Error() }, func(ok bool) { h.done <- ok }) if <-h.done { t.Fatal("the load reported success") } if h.prepared != 0 || len(h.restored) != 0 { t.Fatalf("prepare=%d restore=%d after a refusal, want 0 and 0", h.prepared, len(h.restored)) } if tc.want != nil && !errors.Is(got, tc.want) { t.Fatalf("refused with %v, want %v", got, tc.want) } if b, err := os.ReadFile(kept); err != nil || string(b) != "the household's photo" { t.Fatalf("the kept file changed: %q %v", b, err) } if st := h.m.RestoreStatus(); st.Running || st.Last == nil || st.Last.OK { t.Fatalf("the restore record does not read as failed: %+v", st) } if _, err := os.Stat(h.proofScratch(t, r691App)); !os.IsNotExist(err) { t.Fatalf("the downloaded copy was left behind after a refusal (err=%v)", err) } }) } }