Files
felhom-controller/controller/internal/backup/offbox_reconstitute_test.go
T
admin 08eb1a6e3a
gates / gates (push) Failing after 12s
R-356: the off-site restore refused every app that has no data drive
ReconstituteFromOffsite and PlaceOffsiteRestore both resolved the restore
destination with the RAW HDD_PATH and read an empty answer as "the app is not
installed". For 40 of the 53 catalogue apps that answer is correctly empty and
permanent, so both actions refused forever for a running, healthy app — and told
the customer to reinstall it "in the same place", which those apps never offer.

Separate the two questions. "Installed?" is asked of ListDeployedStacks via a new
Manager.isStackDeployed that fails CLOSED on a nil provider. "Where?" is answered
by GetAppDrivePath — the same resolver CaptureRecoveryUnit wrote the snapshot
with, so the restore aims at the place the backup came from.

The 13 drive apps are unchanged: own drive, mismatch check, ack still required.
A third refusal, with its own sentence, covers installed-but-no-resolvable-root.

Fixtures that marked an app "installed" by giving it an HDD path now state
deployment as its own fact. No assertion weakened.
2026-08-22 13:08:46 +02:00

459 lines
19 KiB
Go

package backup
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// R-43/R-44 (v0.148.0) — the coherent-pair + true-restore tests.
//
// These exist because the product shipped a restore button for months that could not restore.
// DIAG-immich-restore-2026-07-19: 11 photos, files intact, timeline empty, two "successful"
// restores that merged 0 files and never touched postgres. Every test below asserts a behaviour
// whose absence produced that outcome, so each one is a regression guard for a real incident
// rather than a description of the current implementation.
// recordingProvider records stop/start call ORDER so the reconstitution sequence can be asserted.
//
// R-47 widened it: it now also records the DB-ONLY bring-up and, critically, whether a FULL start
// has happened yet — the state the replay must observe as `false`. That single flag is what
// separates the fixed sequence from the one that produced H4, in which the whole stack was already
// up (and rebuilding its own schema) when the dump replay began.
type recordingProvider struct {
offbox3aProvider
calls []string
composePath string // the LIVE compose the DB-service resolver reads
gotServices []string // services passed to StartStackServices
fullStarted bool // a FULL StartStack has happened
startSvcErr error // injected StartStackServices failure
}
func (p *recordingProvider) StopStack(string) error { p.calls = append(p.calls, "stop"); return nil }
func (p *recordingProvider) StartStack(string) error {
p.fullStarted = true
p.calls = append(p.calls, "start")
return nil
}
func (p *recordingProvider) StartStackServices(_ string, services []string) error {
p.gotServices = append([]string(nil), services...)
p.calls = append(p.calls, "startsvc:"+strings.Join(services, ","))
return p.startSvcErr
}
func (p *recordingProvider) GetStackComposePath(string) (string, bool) {
return p.composePath, p.composePath != ""
}
// The app really is up again after StartStack, so the post-restore health wait returns at once.
// Leaving it false would make each test sit through the full 90s deadline.
func (p *recordingProvider) RefreshAndIsRunning(string) bool { return true }
// recoveryProvider adds the recovery info CaptureRecoveryUnit needs (the shared 3a provider has none).
type recoveryProvider struct {
offbox3aProvider
stackDir string
}
func (p *recoveryProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) {
return RecoveryInfo{DisplayName: "Immich", StackDir: p.stackDir}, name == "immich"
}
// pgDump builds a structurally valid postgres dump big enough to clear ValidateDump's 100-byte
// floor, with the accounts-table COPY block carrying `rows` rows. The R-44 sniff runs only on a
// dump that already passes structural validation, so a toy fixture would silently skip it.
func pgDump(rows int) string {
const head = `-- PostgreSQL database dump
-- Dumped from database version 16.10
SET statement_timeout = 0;
SET lock_timeout = 0;
SET client_encoding = 'UTF8';
CREATE TABLE public.asset (id uuid NOT NULL);
CREATE TABLE public."user" (id uuid NOT NULL, email text);
COPY public."user" (id, email) FROM stdin;
`
var b strings.Builder
b.WriteString(head)
for i := 0; i < rows; i++ {
b.WriteString("id-x\tuser@example.invalid\n")
}
b.WriteString("\\.\n") // the COPY-block terminator
return b.String()
}
// reconFixture builds a manager with a COMPLETED full scratch for `immich`, a snapshot whose unit
// carries the given coherence stamp, and injectable copy/dump/import seams.
func reconFixture(t *testing.T, runID, dumpsAt string, dumpBody string) (*Manager, *recordingProvider, *[]string) {
t.Helper()
drive := t.TempDir()
m, sett := newOffboxManager(t)
prov := &recordingProvider{offbox3aProvider: offbox3aProvider{
hdd: map[string]string{"immich": drive}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{},
// R-356: the app is DEPLOYED, stated as its own fact. It used to be implied by owning a
// drive; the restore now asks the two questions separately.
deployed: map[string]bool{"immich": true},
}}
m.SetStackProvider(prov)
if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "USB", Schedulable: true}); err != nil {
t.Fatal(err)
}
// The LIVE compose the reconstitution reads to learn WHICH service holds the database (R-47).
// Immich-shaped on purpose: an app service, a redis service that must never be mistaken for a
// database, and a top-level `volumes:` key whose entry looks exactly like a service to a line scan.
liveStackDir := t.TempDir()
prov.composePath = filepath.Join(liveStackDir, "docker-compose.yml")
if err := os.WriteFile(prov.composePath, []byte(immichLikeCompose), 0o644); err != nil {
t.Fatal(err)
}
scratch, liveNs, err := m.offboxRestoreScratchDir("immich")
if err != nil {
t.Fatal(err)
}
oldNs := "/felhomdata/ns"
unitP := oldNs + "/backups/primary/immich"
dataP := oldNs + "/appdata/immich"
placements, err := mapOffsiteRestorePaths([]string{unitP, dataP}, "immich", scratch, liveNs)
if err != nil {
t.Fatal(err)
}
for _, pl := range placements {
if err := os.MkdirAll(pl.src, 0o755); err != nil {
t.Fatal(err)
}
if pl.isUnit {
dd := filepath.Join(pl.src, "db-dumps")
if err := os.MkdirAll(dd, 0o755); err != nil {
t.Fatal(err)
}
if dumpBody != "" {
if err := os.WriteFile(filepath.Join(dd, "immich-postgres.sql"), []byte(dumpBody), 0o644); err != nil {
t.Fatal(err)
}
}
man := &RecoveryManifest{SchemaVersion: 1, AppName: "immich", OffsiteRunID: runID, DumpsAt: dumpsAt}
if err := writeManifest(filepath.Join(pl.src, "manifest.json"), man); err != nil {
t.Fatal(err)
}
}
}
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
m.SetOffboxSizer(func(string) int64 { return 1 << 20 })
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
if contains(args, "snapshots") {
return []byte(`[{"short_id":"snap1","time":"2026-07-19T06:00:00Z","paths":["` + unitP + `","` + dataP + `"]}]`), nil
}
return nil, nil
})
// Seams: one DB, a safety dump that really writes a file, and a recording importer.
db := DiscoveredDB{StackName: "immich", ContainerName: "immich-postgres", DBType: DBTypePostgres}
m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return []DiscoveredDB{db}, nil }
m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, dir string) DumpResult {
p := filepath.Join(dir, "immich-postgres.sql")
_ = os.MkdirAll(dir, 0o755)
_ = os.WriteFile(p, []byte(pgDump(1)), 0o644)
return DumpResult{DB: d, FilePath: p, Size: 42}
})
var imported []string
m.importDBDump = func(_ context.Context, _ DiscoveredDB, p string) error {
imported = append(imported, p)
return nil
}
m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { return 3, nil })
return m, prov, &imported
}
// TestReconstituteReplaysDBAndOrdersOperations is Scenario C: the whole point of R-43. A restore of
// a DB-indexed app must stop the app, place files, restart it and REPLAY the snapshot's dump — and
// the safety dump must exist before any of it. Before v0.148.0 the replay simply did not happen,
// which is why the photos never came back.
func TestReconstituteReplaysDBAndOrdersOperations(t *testing.T) {
m, prov, imported := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if err != nil {
t.Fatalf("reconstitute: %v", err)
}
if res.DBsReplayed != 1 {
t.Fatalf("expected the snapshot dump to be replayed exactly once, got %d — this is the R-43 defect", res.DBsReplayed)
}
if len(*imported) != 1 || !strings.Contains((*imported)[0], "immich-postgres.sql") {
t.Fatalf("expected an import of the snapshot dump, got %v", *imported)
}
// The dump replayed must come from the SCRATCH unit, never the live one: the live unit is
// deliberately not overwritten, so replaying from it would replay the CURRENT database back over
// itself and restore nothing.
if !strings.Contains((*imported)[0], "offsite-restore") {
t.Fatalf("replay source must be the restored scratch unit, got %s", (*imported)[0])
}
if res.FilesPlaced != 3 {
t.Fatalf("expected the userdata placement to be counted, got %d", res.FilesPlaced)
}
// stop BEFORE the file copy; then ONLY the database service up for the replay (R-47 — a full
// start here is the H4 race); the full start comes last.
if got := strings.Join(prov.calls, ","); got != "stop,startsvc:immich-postgres,start" {
t.Fatalf("expected stop → db-only start → full start around the restore, got %q", got)
}
if res.SafetyDump == "" {
t.Fatal("no safety dump recorded — the undo must exist")
}
if _, err := os.Stat(res.SafetyDump); err != nil {
t.Fatalf("safety dump not on disk: %v", err)
}
if !strings.HasPrefix(filepath.Base(res.SafetyDump), preRestoreDumpPrefix) {
t.Fatalf("safety dump must carry the pre-restore prefix so it is never replayed as a source, got %s", filepath.Base(res.SafetyDump))
}
}
// TestReconstituteRefusesWhenSafetyDumpFails is the RED-PROOF for the undo invariant: a replay whose
// previous state was not captured is an overwrite with no way back, so it must not happen at all —
// and it must abort with the live app untouched (no stop, no copy).
func TestReconstituteRefusesWhenSafetyDumpFails(t *testing.T) {
m, prov, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult {
return DumpResult{DB: d, Error: context.DeadlineExceeded}
})
var copied bool
m.SetOffboxFullPlaceCopier(func(_, _ string) (int, error) { copied = true; return 1, nil })
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if err == nil {
t.Fatal("expected a refusal when the safety dump cannot be taken")
}
if len(*imported) != 0 {
t.Fatalf("REPLAYED WITHOUT AN UNDO — the exact thing the invariant forbids: %v", *imported)
}
if copied {
t.Fatal("files were overwritten despite the refusal — the abort must leave live data untouched")
}
if len(prov.calls) != 0 {
t.Fatalf("the app was stopped despite the refusal, got %v", prov.calls)
}
}
// TestReconstituteNoDBAppMakesNoDumpOrImportCalls is Scenario E: an app without a database must flow
// exactly as before — no safety dump, no replay — so the new leg cannot regress the simple case.
func TestReconstituteNoDBAppMakesNoDumpOrImportCalls(t *testing.T) {
m, _, imported := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "")
m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
dumped := 0
m.SetSafetyDumpFn(func(_ context.Context, d DiscoveredDB, _ string) DumpResult {
dumped++
return DumpResult{DB: d}
})
res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if err != nil {
t.Fatalf("reconstitute: %v", err)
}
if dumped != 0 {
t.Fatalf("a no-DB app must not produce a safety dump, got %d call(s)", dumped)
}
if len(*imported) != 0 {
t.Fatalf("a no-DB app must not import anything, got %v", *imported)
}
if res.SafetyDump != "" || res.DBsReplayed != 0 {
t.Fatalf("unexpected DB activity: safety=%q replayed=%d", res.SafetyDump, res.DBsReplayed)
}
}
// TestReconstituteSurfacesLegacySkewedPair is Scenario D: a pre-v0.148 snapshot carries no coherence
// stamp, so its two halves may be from different times. That must be SURFACED (and reversible), never
// blocked — the customer's own judgement is the gate, and refusing would deny a legitimate restore.
func TestReconstituteSurfacesLegacySkewedPair(t *testing.T) {
m, _, imported := reconFixture(t, "", "", pgDump(1))
res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if err != nil {
t.Fatalf("a legacy pair must still be restorable, got refusal: %v", err)
}
if !res.Skewed {
t.Fatal("an unstamped (pre-v0.148) snapshot must report Skewed so the confirm can say so")
}
if len(*imported) != 1 {
t.Fatalf("the legacy restore must still replay, got %v", *imported)
}
}
// TestReconstituteFlagsCustomerEmptyDump is the R-44 sniff at the restore end: the immich dump that
// started all of this was structurally valid and contained zero users. Restoring it is allowed, but
// the customer must be told before they commit.
func TestReconstituteFlagsCustomerEmptyDump(t *testing.T) {
// A valid postgres dump whose accounts table has NO rows — the 2026-07-19 shape exactly.
m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(0))
res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if err != nil {
t.Fatalf("the sniff must never block a restore: %v", err)
}
if !res.LooksEmpty {
t.Fatal("a dump with an empty accounts table must raise the warn-level signal")
}
}
// TestOffsiteScratchPairReportsWhatTheConfirmNeeds covers the page-render surface: the confirm can
// only be honest if this reports the pair's age and warnings before anything is started.
func TestOffsiteScratchPairReportsWhatTheConfirmNeeds(t *testing.T) {
m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z",
"-- PostgreSQL database dump\nCREATE TABLE a();\nCOPY public.\"user\" (id) FROM stdin;\n7\n\\.\n")
info := m.OffsiteScratchPair("immich")
if !info.Ready || !info.HasDump {
t.Fatalf("expected a ready pair with a dump, got %+v", info)
}
if info.Skewed {
t.Fatal("a stamped snapshot must not be reported as skewed")
}
if info.LooksEmpty {
t.Fatal("a dump with account rows must not be flagged empty")
}
want, _ := time.Parse(time.RFC3339, "2026-07-19T06:00:00Z")
if !info.DumpsAt.Equal(want) {
t.Fatalf("DumpsAt = %v, want %v", info.DumpsAt, want)
}
}
// --- R-44: the coherence pre-phase -----------------------------------------------------------
// TestOffsiteRunDumpsBeforeCapture is Scenarios A + B. The ORDER is the entire mechanism: dumps
// must be refreshed BEFORE restic captures, so the snapshot pairs this run's database with this
// run's files. Reversed, the snapshot would hold rows pointing at files that were never captured.
//
// It also asserts the ordering on the NIGHTLY entry point (RunOffboxBackup, no progress sink), not
// just the manual one — before v0.148.0 the nightly ordering was an accident of two independent
// scheduler entries at 02:30 and 04:15, which a schedule edit could silently invert.
func TestOffsiteRunDumpsBeforeCapture(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
mkUnit(t, drive, "immich")
if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd["immich"] = drive
prov.has["immich"] = true
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
_ = sett.SetAppOffbox("immich", true)
var order []string
m.SetOffsitePreDumpFn(func(context.Context) error {
order = append(order, "dump")
return nil
})
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
switch {
case contains(args, "cat") && contains(args, "config"):
return []byte(`{"version":2}`), nil
case contains(args, "backup"):
order = append(order, "capture")
return nil, nil
case contains(args, "snapshots"):
return []byte(`[]`), nil
case contains(args, "stats"):
return []byte(`{"total_size":123}`), nil
}
return nil, nil
})
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run: %v", err)
}
if len(order) < 2 {
t.Fatalf("expected both a dump and a capture, got %v", order)
}
if order[0] != "dump" {
t.Fatalf("the dump leg MUST precede the capture (R-44); got %v", order)
}
if order[1] != "capture" {
t.Fatalf("expected the capture immediately after the dump, got %v", order)
}
}
// TestOffsiteRunContinuesWhenDumpLegFails is the data-first rule: a dump failure degrades the
// snapshot's DB half but must NOT abort the push. Refusing to ship the files would turn a partial
// backup into no backup at all — strictly worse for the customer.
func TestOffsiteRunContinuesWhenDumpLegFails(t *testing.T) {
drive := t.TempDir()
m, sett, prov := classifiedOffboxManager(t, drive)
mkUnit(t, drive, "immich")
if err := os.MkdirAll(filepath.Join(drive, "appdata", "immich"), 0o755); err != nil {
t.Fatal(err)
}
prov.hdd["immich"] = drive
prov.has["immich"] = true
prov.binds["immich"] = []ClassifiedBind{mandatoryHDD("appdata/immich")}
_ = sett.SetAppOffbox("immich", true)
m.SetOffsitePreDumpFn(func(context.Context) error { return context.DeadlineExceeded })
cap := &backupCapture{}
m.SetOffboxRunner(cap.runner())
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("a dump failure must not fail the whole run: %v", err)
}
if cap.backups != 1 {
t.Fatalf("the files must still be pushed after a dump failure, got %d capture(s)", cap.backups)
}
}
// TestCaptureRecoveryUnitStampsAndCarriesRunID covers the stamp that makes a pair verifiable at
// restore time, and the trap beside it: the PERIODIC refresh must neither invent a coherence claim
// nor erase one a real run established.
func TestCaptureRecoveryUnitStampsAndCarriesRunID(t *testing.T) {
drive := t.TempDir()
m, _, base := classifiedOffboxManager(t, drive)
base.hdd["immich"] = drive
// CaptureRecoveryUnit needs real recovery info + a compose dir to read; the shared fixture
// provider returns none, so wrap it rather than widening a struct four other test files use.
stackDir := filepath.Join(t.TempDir(), "immich")
if err := os.MkdirAll(stackDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stackDir, "docker-compose.yml"), []byte("services: {}\n"), 0o644); err != nil {
t.Fatal(err)
}
m.SetStackProvider(&recoveryProvider{offbox3aProvider: *base, stackDir: stackDir})
// 1) A run in flight stamps the manifest.
end := m.beginOffsiteRunStamp("run-A")
if err := m.CaptureRecoveryUnit("immich"); err != nil {
t.Fatalf("capture: %v", err)
}
end()
man := readManifest(RecoveryUnitManifestPath(drive, "immich"))
if man == nil || man.OffsiteRunID != "run-A" {
t.Fatalf("expected the in-flight run id to be stamped, got %+v", man)
}
if man.DumpsAt == "" {
t.Fatal("a stamped unit must record when its dumps were taken")
}
// 2) A periodic refresh (no run in flight) must CARRY the stamp forward, not blank it — a unit
// that silently lost its stamp would be re-reported as a skewed legacy pair at restore time.
if err := m.CaptureRecoveryUnit("immich"); err != nil {
t.Fatalf("refresh: %v", err)
}
man2 := readManifest(RecoveryUnitManifestPath(drive, "immich"))
if man2 == nil || man2.OffsiteRunID != "run-A" {
t.Fatalf("the periodic refresh erased the coherence stamp: %+v", man2)
}
// 3) A NEW run re-stamps even though nothing else about the unit changed — the idempotent-skip
// must not swallow the one field the restore path reads.
end2 := m.beginOffsiteRunStamp("run-B")
if err := m.CaptureRecoveryUnit("immich"); err != nil {
t.Fatalf("capture 2: %v", err)
}
end2()
man3 := readManifest(RecoveryUnitManifestPath(drive, "immich"))
if man3 == nil || man3.OffsiteRunID != "run-B" {
t.Fatalf("a new run must re-stamp the unit, got %+v", man3)
}
}