felhom-controller/controller/internal/backup/r1340_hold_cause_test.go

package backup

import (
	"context"
	"errors"
	"io"
	"log"
	"os"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed,
// whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail
// (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on
// Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its
// DATABASE restore failed and was rolled back.
//
// RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from
// holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ /
// „kind """.

func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold {
	t.Helper()
	if m.settings == nil {
		t.Fatal("fixture: no settings wired")
	}
	h, ok := m.settings.GetRestoreHold(stack)
	if !ok {
		t.Fatalf("%s is not held", stack)
	}
	return h
}

// The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails.
func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) {
	m, vp, rig, _, notified, _ := r893PutBackFixture(t)
	rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use")
	p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
	m.SetStackProvider(p)
	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
		t.Fatal("a failed restore must be surfaced")
	}
	h := holdOf(t, m, "immich")
	if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack {
		t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause)
	}
	if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") {
		t.Errorf("failed step %q — want the stop and its error", h.FailedStep)
	}
	if h.DBReplay {
		t.Error("a volume failure must not be recorded as a database replay")
	}
	if h.RestoreKind != settings.RestoreKindOffsite {
		t.Errorf("restore kind %q, want offsite", h.RestoreKind)
	}
	if h.SafetyDump == "" {
		t.Error("the kept copy's folder must be recorded")
	} else if _, err := os.Stat(h.SafetyDump); err != nil {
		t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err)
	}
	if *notified != 1 {
		t.Errorf("hold notices %d, want 1", *notified)
	}
}

// The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded.
func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) {
	m, vp, _, _, _, _ := r893PutBackFixture(t)
	m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2})
	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
		t.Fatal("a failed restore must be surfaced")
	}
	h := holdOf(t, m, "immich")
	if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay {
		t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay)
	}
}

// R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved.
func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) {
	m, _, _ := r1256DefFailFixture(t, 2)
	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
		t.Fatal("a failed definition write must be surfaced")
	}
	h := holdOf(t, m, "immich")
	if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay {
		t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay)
	}
	if !strings.Contains(h.FailedStep, "writing the live app definition back failed") {
		t.Errorf("failed step %q", h.FailedStep)
	}
}

// A LOCAL restore (from the app's own backup) records itself as local, never off-site.
func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) {
	m, _, rig, _ := r1177Fixture(t)
	if m.settings == nil {
		sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0))
		if err != nil {
			t.Fatal(err)
		}
		m.settings = sett
	}
	rig.unitErr = errors.New("tar: short read")
	rig.putBackErr = errors.New("tar xf failed")
	if _, err := m.RestoreFromRecoveryUnit("app"); err == nil {
		t.Fatal("a failed put-back must be surfaced")
	}
	h := holdOf(t, m, "app")
	if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay {
		t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay)
	}
	if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") {
		t.Errorf("failed step %q", h.FailedStep)
	}
}

// A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause.
func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) {
	r := newR1251Rig(t)
	r.putErr = errors.New("compose down: timeout")
	r.guard().Recover()
	h, ok := r.sett.GetRestoreHold("immich")
	if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack {
		t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok)
	}
	if !strings.Contains(h.FailedStep, "compose down: timeout") {
		t.Errorf("failed step %q", h.FailedStep)
	}
}