felhom-controller/controller/internal/backup/r893_mixed_restore_test.go

package backup

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)

// R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md).
//
// An off-site restore of one app writes the snapshot's definition (when the version differs), copies the
// snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails,
// the rollback puts back only the database rows. Until this slice the app was started on that mix —
// an older definition, the snapshot's files and volumes, the newer database — and the household was told
// „your data is back as it was before the restore". That is true only of the database rows.
//
// Now: the database is still rolled back, the LIVE definition is written back, and the app is HELD
// stopped with a sentence that says it needs support. Pinned here:
//   (a) the definition write is called a second time, with the live definition;
//   (b) the app is NOT started;
//   (c) a hold is left (kind restore_mixed), and the operator is notified;
//   (d) neither the error nor the hold sentence claims the data is back as it was (hu and en).
// The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA.

// r893Provider records EVERY definition write (vtReconProvider keeps only the last).
type r893Provider struct {
	*vtReconProvider
	defs   [][]string
	onStop func()
}

func (p *r893Provider) StopStack(name string) error {
	if p.onStop != nil {
		p.onStop()
	}
	return p.vtReconProvider.StopStack(name)
}

func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error {
	p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml")))
	return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env)
}

func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) {
	t.Helper()
	m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1))
	m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") }
	rolled := 0
	m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })
	notified := 0
	m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) {
		notified++
		if replayErr == nil || rollbackErr != nil {
			t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr)
		}
	})
	m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil }
	live := "16"
	if versionChange {
		live = "18"
	}
	// The LIVE definition, on disk where the app runs.
	if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil {
		t.Fatal(err)
	}
	vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}}
	m.SetStackProvider(vp)
	vtSnapshotUnit(t, m, "16", true)
	return m, vp, &rolled, &notified
}

func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) {
	t.Helper()
	if err == nil {
		t.Fatal("a failed replay must be surfaced as a failure")
	}
	if rolled != 1 {
		t.Fatalf("the database rollback must still run exactly once, ran %d", rolled)
	}
	// (b)
	if vp.fullStarted {
		t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls)
	}
	if last := vp.calls[len(vp.calls)-1]; last != "stop" {
		t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls)
	}
	// (c)
	held, sentence := m.RestoreHoldFor("immich")
	if !held {
		t.Fatal("no hold was left — every start path would start the app on the mixed state")
	}
	if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed {
		t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed)
	}
	if notified != 1 {
		t.Errorf("the operator must be notified once, got %d", notified)
	}
	// (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too).
	for _, s := range []string{err.Error(), sentence} {
		low := strings.ToLower(s)
		if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") {
			t.Errorf("the sentence still claims the data is back / the app runs: %q", s)
		}
		if !strings.Contains(low, "kapcsolatot") {
			t.Errorf("the sentence must send the household to support: %q", s)
		}
	}
	_, en := m.RestoreHoldForLang("immich", "en")
	if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") {
		t.Errorf("English hold sentence: %q", en)
	}
	if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") {
		t.Errorf("English error sentence: %q", enErr)
	}
}

func TestR893_AVersionChangeThenAFailedReplayHoldsTheAppAtItsLiveDefinition(t *testing.T) {
	m, vp, rolled, notified := r893Fixture(t, true, 0)
	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	assertR893Held(t, m, vp, err, *rolled, *notified)
	// (a)
	if len(vp.defs) != 2 {
		t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs)
	}
	if got := strings.Join(vp.defs[1], " "); !strings.Contains(got, "postgres:18-alpine") {
		t.Fatalf("the definition written back is %q — want the live 18", got)
	}
}

func TestR893_AReplacedVolumeThenAFailedReplayHoldsTheApp(t *testing.T) {
	m, vp, rolled, notified := r893Fixture(t, false, 1)
	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	assertR893Held(t, m, vp, err, *rolled, *notified)
	if len(vp.defs) != 0 {
		t.Fatalf("no version changed, yet a definition was written: %v", vp.defs)
	}
}

// Security review 2026-10-08 (G1): the hold covers EVERY failure after the snapshot's definition is written, not only a
// failed replay. Here the DB-only start fails after a version change: before the fix the app was started at the
// snapshot's (older) definition on the live database. Now the live definition is written back and the app is held.
// RED-PROOF: restore `restartStack()` as the only action in the StartStackServices failure branch → started → FAILS.
func TestR893_AVersionChangeThenAFailedDBStartHoldsTheApp(t *testing.T) {
	m, vp, rolled, notified := r893Fixture(t, true, 0)
	vp.startSvcErr = context.DeadlineExceeded
	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if err == nil {
		t.Fatal("a failed DB-only start must be surfaced")
	}
	if vp.fullStarted {
		t.Fatalf("the app was STARTED at the snapshot's definition on the live data — calls %v", vp.calls)
	}
	if held, _ := m.RestoreHoldFor("immich"); !held {
		t.Fatal("no hold was left after a failed DB-only start that followed a version change")
	}
	if len(vp.defs) != 2 || !strings.Contains(strings.Join(vp.defs[1], " "), "postgres:18-alpine") {
		t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs)
	}
	if *rolled != 0 || *notified != 1 {
		t.Errorf("rollback ran %d (want 0: nothing was replayed), notified %d (want 1)", *rolled, *notified)
	}
}

// Security review 2026-10-08 (G3): the hold is persisted BEFORE the app is stopped, so a controller that dies in
// between cannot restart the app from its app-stop marker with nothing refusing it. The provider's StopStack asserts
// the hold is already on disk at the moment it is called.
// RED-PROOF: move SetRestoreHold after StopStack in holdAppAfterMixedRestore → FAILS.
func TestR893_HoldIsPersistedBeforeTheStop(t *testing.T) {
	m, vp, _, _ := r893Fixture(t, true, 0)
	seenAtStop := []bool{}
	vp.onStop = func() {
		held, _ := m.RestoreHoldFor("immich")
		seenAtStop = append(seenAtStop, held)
	}
	_, _ = m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if len(seenAtStop) == 0 || !seenAtStop[len(seenAtStop)-1] {
		t.Fatalf("the last stop ran before the hold was persisted (held at each stop: %v)", seenAtStop)
	}
}