felhom-controller/controller/internal/backup/r379_rollback_test.go

package backup

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// R-379 / R-380. When an off-site database replay failed, the app was restarted onto a HALF-WRITTEN
// database and the customer was shown the undo copy's filename — a file nothing in the product could
// apply. Measured live on demo-hp 2026-08-22: Postgres left emptied and crash-looping, MariaDB left
// partly applied while `docker inspect` reported health=healthy. Both are the same failure — a half
// state — and the only difference was whether it looked broken.
//
// The fix puts the customer's own pre-restore copy back automatically. These tests pin that, the
// double-failure hold, and the two absence claims that go with them.

// rollbackFixture is reconFixture with the replay failing and the rollback injectable.
func rollbackFixture(t *testing.T, rollbackErr error) (*Manager, *recordingProvider, *int) {
	t.Helper()
	m, prov, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
	m.importDBDump = func(context.Context, DiscoveredDB, string) error {
		return errors.New("replay blew up")
	}
	calls := 0
	m.SetRollbackImportFn(func(_ context.Context, _ DiscoveredDB, path string) error {
		calls++
		// The rollback must be handed THIS RUN's undo file, not the scratch's dump.
		if !strings.Contains(filepath.Base(path), preRestoreDumpPrefix) {
			t.Errorf("rollback was handed %q — that is not an undo copy", filepath.Base(path))
		}
		return rollbackErr
	})
	return m, prov, &calls
}

// SCENARIO A — replay fails, rollback succeeds. The app comes back and the outcome records it.
//
// WRONG OUTCOMES PINNED: the app left down; or a message that reports the failure and omits that the
// data is back — the omission of a GAIN is as misleading as the omission of a loss, and here it is
// the difference between a customer who panics and one who does not.
func TestR379_ScenarioA_RollbackSucceeds_AppComesBack(t *testing.T) {
	m, prov, calls := rollbackFixture(t, nil)

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("a failed replay must still be surfaced as a failure")
	}
	if *calls != 1 {
		t.Fatalf("the undo copy must be re-applied exactly once, got %d", *calls)
	}
	if !res.RolledBack {
		t.Error("the result must record that the rollback happened")
	}
	if !prov.fullStarted {
		t.Fatal("after a successful rollback the app must be started — it is in a known good state")
	}
	// The customer sentence says BOTH things.
	low := err.Error()
	if !strings.Contains(low, "sikertelen") {
		t.Errorf("the message must say the restore failed; got: %v", err)
	}
	if !strings.Contains(low, "visszakerültek") {
		t.Errorf("the message must say the data is back as it was; got: %v", err)
	}
	// And no hold was written — the app is fine.
	if held, _ := m.RestoreHoldFor("immich"); held {
		t.Error("a recovered app must NOT be held")
	}
}

// SCENARIO C — replay fails AND rollback fails. The app is held, not started.
//
// OPERATOR RULING 2026-08-22: a running app on a half-written database lets the customer type into
// it and makes the damage permanent.
//
// WRONG OUTCOMES PINNED: the app started anyway; or the app-stop marker left active, which would
// have Recover() start the broken app at the next controller boot, quietly, hours later.
func TestR379_ScenarioC_BothFail_AppIsHeldNotStarted(t *testing.T) {
	m, prov, calls := rollbackFixture(t, errors.New("rollback blew up too"))
	notified := 0
	m.SetRestoreHoldNotify(func(string, error, error) { notified++ })

	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("a double failure must be surfaced")
	}
	if *calls != 1 {
		t.Fatalf("the rollback must have been attempted, got %d calls", *calls)
	}
	// THE OBSERVABLE THAT MATTERS.
	if prov.fullStarted {
		t.Fatal("the app was STARTED onto a half-written database — the exact outcome the hold exists to prevent")
	}
	held, why := m.RestoreHoldFor("immich")
	if !held {
		t.Fatal("the hold must be persisted, or nothing will refuse a restart later")
	}
	if why == "" || !strings.Contains(why, "kapcsolat") {
		t.Errorf("the hold's reason must name a route the customer can take; got %q", why)
	}
	if notified != 1 {
		t.Errorf("the operator must be told exactly once, got %d", notified)
	}
	if !strings.Contains(err.Error(), "LEÁLLÍTVA") {
		t.Errorf("the customer must be told the app was deliberately stopped; got: %v", err)
	}
}

// SCENARIO E — the replay SUCCEEDS. Nothing new may happen.
//
// WRONG OUTCOME PINNED: a rollback firing on a successful restore would overwrite the restored data
// with the pre-restore state — a silent, total loss of the thing the customer asked for.
func TestR379_ScenarioE_SuccessfulReplay_NoRollbackNoHold(t *testing.T) {
	m, prov, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
	rolled := 0
	m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })
	held := 0
	m.SetRestoreHoldNotify(func(string, error, error) { held++ })

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("a clean restore must succeed: %v", err)
	}
	if rolled != 0 {
		t.Fatalf("a rollback fired on a SUCCESSFUL restore — the restored data would be overwritten; calls=%d", rolled)
	}
	if res.RolledBack {
		t.Error("a successful restore must not report a rollback")
	}
	if held != 0 {
		t.Errorf("no hold may be written on success, got %d", held)
	}
	if isHeld, _ := m.RestoreHoldFor("immich"); isHeld {
		t.Error("a successful restore must leave no hold")
	}
	if !prov.fullStarted {
		t.Error("a successful restore still starts the app")
	}
}

// SCENARIO E, POSITIVE CONTROL. "No rollback fired" is an absence claim, so prove the counter can
// count: the same seam, driven through the failure path, must register.
func TestR379_ScenarioE_PositiveControl_TheCounterCanCount(t *testing.T) {
	m, _, calls := rollbackFixture(t, nil)
	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
		t.Fatal("fixture: the replay must fail here")
	}
	if *calls == 0 {
		t.Fatal("the rollback counter never increments — Scenario E's zero would have proven nothing")
	}
}

// SCENARIO F — an app with no database. Untouched in every respect.
func TestR379_ScenarioF_NoDatabase_Unchanged(t *testing.T) {
	m, prov, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "")
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
	rolled := 0
	m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })

	res, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err != nil {
		t.Fatalf("a no-DB app must restore: %v", err)
	}
	if rolled != 0 || res.RolledBack {
		t.Error("a no-DB app has nothing to undo and nothing to roll back")
	}
	if res.SafetyDump != "" {
		t.Errorf("a no-DB app takes no undo copy, got %q", res.SafetyDump)
	}
	if held, _ := m.RestoreHoldFor("immich"); held {
		t.Error("a no-DB app must never be held")
	}
	if !prov.fullStarted {
		t.Error("a no-DB app still starts")
	}
}

// SCENARIO G — two databases, one replay fails. The WHOLE undo set is re-applied.
//
// WRONG OUTCOME PINNED: only the first. writeSafetyDump used to return one path for an app with two
// databases, so a rollback built on that value would restore one and leave the other half-written —
// this defect, one database over.
func TestR379_ScenarioG_TwoDatabases_WholeSetRolledBack(t *testing.T) {
	m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
	two := []DiscoveredDB{
		{StackName: "immich", DBType: DBTypePostgres, ContainerName: "immich-postgres", ContainerID: "a"},
		{StackName: "immich", DBType: DBTypeMariaDB, ContainerName: "immich-maria", ContainerID: "b"},
	}
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return two, nil }
	m.safetyDumpFn = func(_ context.Context, db DiscoveredDB, dumpDir string) DumpResult {
		p := filepath.Join(dumpDir, "immich-"+string(db.DBType)+".sql")
		if err := os.WriteFile(p, []byte(pgDump(1)), 0o644); err != nil {
			t.Fatal(err)
		}
		return DumpResult{DB: db, FilePath: p, Size: 42}
	}
	m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay failed") }

	var rolled []string
	m.SetRollbackImportFn(func(_ context.Context, db DiscoveredDB, path string) error {
		rolled = append(rolled, db.ContainerName+":"+filepath.Base(path))
		return nil
	})

	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
		t.Fatal("the replay failure must surface")
	}
	if len(rolled) != 2 {
		t.Fatalf("BOTH databases must be rolled back, got %d: %v", len(rolled), rolled)
	}
	// Each database got its OWN undo file, matched by identity rather than by prefix.
	if !strings.Contains(rolled[0], "immich-postgres:") || !strings.Contains(rolled[1], "immich-maria:") {
		t.Errorf("each database must get its own undo copy, got %v", rolled)
	}
	for _, r := range rolled {
		if !strings.Contains(r, preRestoreDumpPrefix) {
			t.Errorf("rollback used a non-undo file: %s", r)
		}
	}
}

// The undo set is matched on THIS RUN's stamp. Three undo copies from three different runs
// accumulated on docmost in one afternoon; a prefix match would replay an arbitrary older state.
func TestR379_UndoSetIsThisRunOnly(t *testing.T) {
	nsRoot := t.TempDir()
	m := newSafetyTestManager()
	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
		return []DiscoveredDB{{StackName: "app", DBType: DBTypePostgres, ContainerName: "app-postgres", ContainerID: "c"}}, nil
	}
	m.safetyDumpFn = func(_ context.Context, db DiscoveredDB, dumpDir string) DumpResult {
		p := filepath.Join(dumpDir, "app-postgres.sql")
		if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
			t.Fatal(err)
		}
		return DumpResult{DB: db, FilePath: p, Size: 1}
	}
	// An OLDER undo copy from a previous run, already on disk.
	dumpDir := AppDBDumpPath(nsRoot, "app")
	if err := os.MkdirAll(dumpDir, 0o755); err != nil {
		t.Fatal(err)
	}
	stale := filepath.Join(dumpDir, preRestoreDumpPrefix+"20200101T000000Z-app-postgres.sql")
	if err := os.WriteFile(stale, []byte("STALE"), 0o644); err != nil {
		t.Fatal(err)
	}

	set, err := m.writeSafetyDump(context.Background(), "app", nsRoot)
	if err != nil {
		t.Fatal(err)
	}
	if len(set.Files) != 1 {
		t.Fatalf("one database → one undo file, got %d", len(set.Files))
	}
	if strings.Contains(set.Files[0].Path, "20200101") {
		t.Fatal("the set picked up an OLDER run's undo copy — it must carry only this run's stamp")
	}
	if set.Stamp == "" || strings.Contains(set.Files[0].Path, set.Stamp) == false {
		t.Errorf("the file must carry this run's stamp %q, got %q", set.Stamp, set.Files[0].Path)
	}
}

// pruneUndoCopies keeps the newest N and never the oldest — ordered by the STAMP, not by mtime.
func TestR379_PruneKeepsTheNewestByStamp(t *testing.T) {
	m := newSafetyTestManager()
	dir := t.TempDir()
	stamps := []string{"20260101T000000Z", "20260201T000000Z", "20260301T000000Z", "20260401T000000Z", "20260501T000000Z"}
	for _, st := range stamps {
		if err := os.WriteFile(filepath.Join(dir, preRestoreDumpPrefix+st+"-app-postgres.sql"), []byte("x"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	// The app's OWN dump must never be a candidate.
	own := filepath.Join(dir, "app-postgres.sql")
	if err := os.WriteFile(own, []byte("own"), 0o644); err != nil {
		t.Fatal(err)
	}

	m.pruneUndoCopies(dir, "app")

	if _, err := os.Stat(own); err != nil {
		t.Fatal("the app's own dump was pruned — only undo copies are candidates")
	}
	for _, st := range stamps[len(stamps)-maxUndoCopiesPerApp:] {
		if _, err := os.Stat(filepath.Join(dir, preRestoreDumpPrefix+st+"-app-postgres.sql")); err != nil {
			t.Errorf("the newest %d must survive; %s is gone", maxUndoCopiesPerApp, st)
		}
	}
	for _, st := range stamps[:len(stamps)-maxUndoCopiesPerApp] {
		if _, err := os.Stat(filepath.Join(dir, preRestoreDumpPrefix+st+"-app-postgres.sql")); err == nil {
			t.Errorf("the oldest must be pruned; %s survived", st)
		}
	}
}

// R-381 — the customer sentence must not carry the engine's output.
//
// MEASURED 2026-08-22: 407 bytes (Postgres, with a caret diagram and `exit status 3`) and 615 bytes
// (MariaDB, whose middle was an `INSERT INTO migrations VALUES (...)` listing — ROWS OUT OF THE
// CUSTOMER'S OWN DATABASE, HTML-escaped, on their dashboard).
func TestR381_CustomerMessageCarriesNoEngineOutput(t *testing.T) {
	engineNoise := "ERROR:  syntax error at end of input\nLINE 1: COPY public.felhom_r356b_discriminator \n  ^ — exit status 3"
	m, _, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", pgDump(1))
	m.importDBDump = func(context.Context, DiscoveredDB, string) error {
		return errors.New(engineNoise)
	}
	m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { return nil })

	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("the failure must surface")
	}
	msg := err.Error()
	for _, leak := range []string{"ERROR:  syntax", "LINE 1:", "COPY public.", "exit status"} {
		if strings.Contains(msg, leak) {
			t.Errorf("the customer message leaks engine output %q; got: %s", leak, msg)
		}
	}
	if len(msg) > 320 {
		t.Errorf("the customer message is %d bytes — it is a sentence, not a transcript: %s", len(msg), msg)
	}
	// POSITIVE CONTROL: the noise really was in the error the code received, so the absence above is
	// the message being clean rather than the noise never existing.
	if !strings.Contains(engineNoise, "exit status") {
		t.Fatal("fixture: the planted noise does not contain the marker this test greps for")
	}
}