felhom-controller/controller/internal/backup/r1251_putback_waits_test.go

package backup

// R-1251 (review round 2, HUNT-1 H1, H2, H7): a pending put-back blocks every start. The first test is the review's
// lean-clone proof (audits/review-round-2-2026-10-24/proofs/H1-drive-cut-restore/) turned round: the same cut, the same
// unbound drive, and now the app is HELD until the copy is poured back, and a later start pours nothing.
//
// RED-PROOF (2026-10-24, each seen failing on main 2449223 before appstop_putback_wait.go):
//   - TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart: „boot 1 must HOLD the app as waiting" —
//     Recover read the copy as gone and only REFUSED the start, with no hold;
//   - TestR1251_AFailedPutBackHoldsTheApp: „a failed put-back must leave a restore_mixed hold" — none was written;
//   - TestR1251_StopPressedDuringTheRestoreStillPutsBack: „the put-back must run" — R-721's skip came first.

import (
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// r1251Rig is one app (immich) whose restore was cut after its copy was recorded, the copy on the app's drive.
type r1251Rig struct {
	marker, drive, undo string
	sett                *settings.Settings
	puts                []string
	starts              []string
	stopped             map[string]bool
	putErr              error
}

func newR1251Rig(t *testing.T) *r1251Rig {
	t.Helper()
	root := t.TempDir()
	r := &r1251Rig{
		marker:  filepath.Join(root, "appstop-state.json"),
		drive:   filepath.Join(root, "drive"),
		stopped: map[string]bool{},
	}
	r.undo = filepath.Join(r.drive, "felhom-data", "backups", "restore-undo", "immich", "20261024T120000Z")
	mustWrite(t, filepath.Join(r.undo, "undo.json"), `{"stack":"immich","copied":["immich_pgdata"]}`)
	sett, err := settings.Load(filepath.Join(root, "settings.json"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatal(err)
	}
	r.sett = sett
	g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
	if err := g.Begin("unit-restore:immich", ReasonUnitRestore, []string{"immich"}); err != nil {
		t.Fatal(err)
	}
	if err := g.SetUndo("immich", r.undo); err != nil {
		t.Fatal(err)
	}
	return r // the power cut: no End
}

func (r *r1251Rig) unbind(t *testing.T) {
	t.Helper()
	if err := os.Rename(r.drive, r.drive+".unbound"); err != nil {
		t.Fatal(err)
	}
}

func (r *r1251Rig) bind(t *testing.T) {
	t.Helper()
	if err := os.Rename(r.drive+".unbound", r.drive); err != nil {
		t.Fatal(err)
	}
}

func (r *r1251Rig) driveBound() bool { _, err := os.Stat(r.drive); return err == nil }

// r1251Starter is the gated starter: it refuses while the drive is absent or a hold is set (as driveStartGate does).
type r1251Starter struct{ r *r1251Rig }

func (s r1251Starter) StartStack(name string) error {
	if _, held := s.r.sett.GetRestoreHold(name); held {
		return ErrStartRefused
	}
	if !s.r.driveBound() {
		return ErrStartRefused
	}
	s.r.starts = append(s.r.starts, name)
	return nil
}
func (s r1251Starter) WantsStopped(name string) bool { return s.r.stopped[name] }

type r1251PutBacker struct{ r *r1251Rig }

func (p r1251PutBacker) PutBackInterruptedRestore(stack, dir string) error {
	if p.r.putErr != nil {
		return p.r.putErr
	}
	p.r.puts = append(p.r.puts, stack)
	return os.RemoveAll(dir) // the real put-back removes the used copy
}

// guard is a fresh controller start over the rig's disk, wired as main.go wires it.
func (r *r1251Rig) guard() *AppStopGuard {
	g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
	g.SetStarter(r1251Starter{r})
	g.SetPutBacker(r1251PutBacker{r})
	g.SetHolds(r.sett)
	g.SetUndoReachable(func(string, string) bool { return r.driveBound() })
	return g
}

func (r *r1251Rig) markerActive() bool {
	b, err := os.ReadFile(r.marker)
	return err == nil && strings.Contains(string(b), `"active": true`)
}

func TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart(t *testing.T) {
	r := newR1251Rig(t)

	// boot 1: the drive is not bound yet
	r.unbind(t)
	g1 := r.guard()
	rec := g1.Recover()
	h, held := r.sett.GetRestoreHold("immich")
	if rec == nil || len(rec.Waiting) != 1 || !held || h.Reason != settings.HoldReasonPutBackPending {
		t.Fatalf("boot 1 must HOLD the app as waiting for its drive: rec=%+v hold=%+v held=%v", rec, h, held)
	}
	if h.SafetyDump != r.undo {
		t.Fatalf("the hold must name the copy, got %q", h.SafetyDump)
	}
	if len(r.starts) != 0 || len(r.puts) != 0 || rec.Alarming() {
		t.Fatalf("nothing may start or be poured while the drive is absent: starts=%v puts=%v alarming=%v", r.starts, r.puts, rec.Alarming())
	}
	if !r.markerActive() {
		t.Fatal("the marker must be kept while the put-back waits")
	}
	// a resume while the drive is still absent does nothing and keeps the hold
	if g1.ResumePutBack("immich") {
		t.Fatal("ResumePutBack must say still-held while the copy cannot be reached")
	}
	if _, still := r.sett.GetRestoreHold("immich"); !still {
		t.Fatal("the hold must stay while the drive is absent")
	}

	// the drive binds: the boot recreate (and the drive-return sweep) call ResumePutBack before they start the app
	r.bind(t)
	if !g1.ResumePutBack("immich") {
		t.Fatal("with the drive bound the put-back must run and allow the start")
	}
	if strings.Join(r.puts, ",") != "immich" {
		t.Fatalf("the copy must be poured back exactly once, puts=%v", r.puts)
	}
	if _, still := r.sett.GetRestoreHold("immich"); still {
		t.Fatal("the waiting hold must be lifted after the put-back")
	}
	if r.markerActive() {
		t.Fatal("the marker must let go of the app after its put-back")
	}
	if g1.ResumePutBack("immich") != true || len(r.puts) != 1 {
		t.Fatalf("a second resume must pour nothing, puts=%v", r.puts)
	}

	// the household writes on the drive, then any later controller start: nothing is poured back
	householdWrite := filepath.Join(r.drive, "felhom-data", "new-photo-after-the-put-back")
	mustWrite(t, householdWrite, "x")
	if rec2 := r.guard().Recover(); rec2 != nil {
		t.Fatalf("a later start must find nothing to recover, got %+v", rec2)
	}
	if len(r.puts) != 1 {
		t.Fatalf("a later start poured the copy again over the household's writes: puts=%v", r.puts)
	}
	if _, err := os.Stat(householdWrite); err != nil {
		t.Fatal("the household's newest file must still be there")
	}
}

// Every start path is refused while the put-back waits: they all read the ONE hold store. The night chain and the
// household's buttons read it through RestoreHoldFor (isHeld, the router's restoreHoldFor, the update guards); the
// boot sweep and Recover through driveStartGate's GetRestoreHold; the web paths (boot recreate, drive return, media
// restart) through appHeld. The web and main.go wiring is pinned in their own packages.
func TestR1251_TheWaitingHoldRefusesEveryStartPath(t *testing.T) {
	r := newR1251Rig(t)
	r.unbind(t)
	r.guard().Recover()
	m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
	held, why := m.RestoreHoldForLang("immich", "hu")
	if !held || !strings.Contains(why, "meghajt") || !strings.Contains(why, "immich") {
		t.Fatalf("the household's sentence must name the app and the drive: held=%v %q", held, why)
	}
	if _, en := m.RestoreHoldForLang("immich", "en"); !strings.Contains(en, "drive is back") {
		t.Fatalf("the English sentence: %q", en)
	}
	if !m.isHeld("immich") {
		t.Fatal("the night chain's isHeld must see the waiting put-back")
	}
	// the gated starter (the boot sweep's and Recover's path) refuses too
	if err := (r1251Starter{r}).StartStack("immich"); !errors.Is(err, ErrStartRefused) {
		t.Fatalf("the gated start must be refused while held, got %v", err)
	}
	// a removal ends the install the copy belongs to: the waiting hold goes with it (a later install must not wait)
	if cleared, _ := r.sett.ClearUpdateHold("immich"); !cleared {
		t.Fatal("a removal must clear the waiting put-back hold")
	}
}

// H2: a put-back that fails at start leaves the app HELD (the D20 hold, „stopped half-way"), never stopped-and-unheld;
// the copy stays and the marker lets go (the hold owns the app).
func TestR1251_AFailedPutBackHoldsTheApp(t *testing.T) {
	r := newR1251Rig(t)
	r.putErr = errors.New("tar: short read")
	rec := r.guard().Recover()
	h, held := r.sett.GetRestoreHold("immich")
	if !held || h.Reason != settings.HoldReasonRestoreMixed {
		t.Fatalf("a failed put-back must leave a restore_mixed hold, got %+v held=%v", h, held)
	}
	if rec == nil || len(rec.Held) != 1 || !rec.Alarming() || len(r.starts) != 0 {
		t.Fatalf("held, alarming, not started: rec=%+v starts=%v", rec, r.starts)
	}
	if !strings.Contains(h.ReplayError, "short read") || h.SafetyDump != r.undo {
		t.Fatalf("the hold must carry the cause and the copy: %+v", h)
	}
	if _, err := os.Stat(r.undo); err != nil {
		t.Fatal("the copy must stay on disk for support")
	}
	if r.markerActive() {
		t.Fatal("the hold owns the app now — the marker must let go, so no later start pours an old copy over it")
	}
	// the household's Start, the boot recreate and the update are all refused by that hold
	m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
	if held, why := m.RestoreHoldForLang("immich", "hu"); !held || !strings.Contains(why, "immich") {
		t.Fatalf("the D20 sentence: held=%v %q", held, why)
	}
}

// A failed resume (the drive came back, the put-back broke) holds the app the same way.
func TestR1251_AFailedResumeHoldsTheApp(t *testing.T) {
	r := newR1251Rig(t)
	r.unbind(t)
	g := r.guard()
	g.Recover()
	r.bind(t)
	r.putErr = errors.New("disk full")
	if g.ResumePutBack("immich") {
		t.Fatal("a failed put-back must not allow the start")
	}
	if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
		t.Fatalf("a failed resume must hold for support, got %+v", h)
	}
	if r.markerActive() {
		t.Fatal("the marker must let go after the hold took over")
	}
}

// A copy that vanished while its drive is BOUND, during a waiting put-back, cannot be poured: held for support.
func TestR1251_ACopyGoneWhileItsDriveIsBoundHoldsTheApp(t *testing.T) {
	r := newR1251Rig(t)
	r.unbind(t)
	g := r.guard()
	g.Recover()
	r.bind(t)
	if err := os.RemoveAll(r.undo); err != nil {
		t.Fatal(err)
	}
	if g.ResumePutBack("immich") {
		t.Fatal("a vanished copy must not allow the start")
	}
	if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
		t.Fatalf("held for support, got %+v", h)
	}
}

// H7: Stop pressed during the restore — the copy is still poured back after the cut; only the start is skipped.
func TestR1251_StopPressedDuringTheRestoreStillPutsBack(t *testing.T) {
	r := newR1251Rig(t)
	r.stopped["immich"] = true
	rec := r.guard().Recover()
	if strings.Join(r.puts, ",") != "immich" {
		t.Fatalf("the put-back must run even though the household pressed Stop, puts=%v rec=%+v", r.puts, rec)
	}
	if len(r.starts) != 0 {
		t.Fatalf("the household stopped it — it must not start, starts=%v", r.starts)
	}
	if r.markerActive() {
		t.Fatal("nothing is owed any more — the marker must clear")
	}
}

// The drive returns later (not at boot): the sweep puts back, then starts through the gated starter — unless the
// household stopped the app.
func TestR1251_ADriveThatReturnsLaterIsPutBackThenStarted(t *testing.T) {
	for _, stopped := range []bool{false, true} {
		r := newR1251Rig(t)
		r.unbind(t)
		g := r.guard()
		g.Recover()
		if got := g.ResumeWaitingPutBacks(); len(got) != 0 || len(r.puts) != 0 {
			t.Fatalf("drive still absent: nothing may happen, started=%v puts=%v", got, r.puts)
		}
		r.bind(t)
		r.stopped["immich"] = stopped
		got := g.ResumeWaitingPutBacks()
		if len(r.puts) != 1 {
			t.Fatalf("stopped=%v: the put-back must run once, puts=%v", stopped, r.puts)
		}
		if want := !stopped; (len(got) == 1) != want || (len(r.starts) == 1) != want {
			t.Fatalf("stopped=%v: started=%v starts=%v", stopped, got, r.starts)
		}
	}
}

// A later restore that completes drops the waiting put-back: the household chose that restore's data.
func TestR1251_ACompletedRestoreDropsTheWaitingPutBack(t *testing.T) {
	r := newR1251Rig(t)
	r.unbind(t)
	r.guard().Recover()
	m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
	m.clearUpdateHoldAfterRestore("immich")
	if _, held := r.sett.GetRestoreHold("immich"); held {
		t.Fatal("a completed restore must drop the waiting put-back hold")
	}
}

// MEASURED on Tester 1 (2026-10-24, audits/restore-holes-2026-10-24/P1-tester1/): a power cut 2.1 s after the copy was
// recorded left its journal as undo.json.tmp — the marker named a copy the start could not read. The copy's folders
// must be synced BEFORE the marker names it. RED-PROOF: with the syncUndoDirs call removed from persistUndoForCrash this
// fails on „no folder was synced before the marker named the copy".
func TestR1251_TheCopyIsOnDiskBeforeTheMarkerNamesIt(t *testing.T) {
	root := t.TempDir()
	marker := filepath.Join(root, "appstop-state.json")
	copyDir := filepath.Join(root, "drive", "backups", "restore-undo", "app", "20261024T171928Z")
	if err := os.MkdirAll(copyDir, 0o700); err != nil {
		t.Fatal(err)
	}
	g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
	if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil {
		t.Fatal(err)
	}
	m := &Manager{logger: log.New(io.Discard, "", 0), appStop: g}
	var synced []string
	markerNamedCopyAtSync := false
	orig := syncUndoDirs
	t.Cleanup(func() { syncUndoDirs = orig })
	syncUndoDirs = func(dir string) error {
		b, _ := os.ReadFile(marker)
		markerNamedCopyAtSync = strings.Contains(string(b), dir)
		synced = append(synced, dir)
		return orig(dir)
	}
	m.persistUndoForCrash("app", &liveVolumeCopy{dir: copyDir, copied: []string{"app_data"}})
	if len(synced) != 1 || synced[0] != copyDir {
		t.Fatalf("no folder was synced before the marker named the copy: %v", synced)
	}
	if markerNamedCopyAtSync {
		t.Fatal("the marker already named the copy when its folders were synced — the order is the fix")
	}
	if b, _ := os.ReadFile(marker); !strings.Contains(string(b), copyDir) {
		t.Fatal("the copy must still be recorded in the marker")
	}
	if _, err := os.Stat(filepath.Join(copyDir, "undo.json")); err != nil {
		t.Fatal("the journal must be in place")
	}
}