felhom-controller/controller/internal/backup/restore_verdict_test.go

package backup

import (
	"context"
	"errors"
	"os"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)

// R-1270 (brief 2026-10-25 Part 3) — the verdict follows the app. Measured before (9202, 0.342.0 = `e1613ae` for these
// files, `audits/restore-truth-2026-10-25/before/`): the Tandoor restore whose app never came up was recorded `ok: true`,
// „…visszaállítva — az alkalmazás újraindult."
// COMPANION RED-PROOF (REPORT.md): with the health failure back to a WARN (restoreNotStarted not called),
// TestR1270_AUnitRestoreWhoseAppDoesNotStartIsNotASuccess and TestR1270_TheOffsiteRestoreToo fail — the restore
// returns nil and the copy is removed.

// r1270Provider is the D22 rig's provider whose app reaches running only when told to (the late start).
type r1270Provider struct {
	*r1177Provider
	up atomic.Bool
}

func (p *r1270Provider) RefreshAndIsRunning(string) bool { return p.up.Load() }

func r1270Rig(t *testing.T) (*Manager, *r1270Provider, *r1177Rig, *[]string) {
	t.Helper()
	m, p, rig, alarms := r1177Fixture(t)
	np := &r1270Provider{r1177Provider: p}
	m.stackProvider = np
	var got []string
	m.SetRestoreNotStartedNotify(func(stack, kind string, cause error) { got = append(got, stack+" "+kind) })
	_ = alarms
	return m, np, rig, &got
}

// Health fails → not ok, the household's own sentence, the undo copy STAYS, one operator alarm.
func TestR1270_AUnitRestoreWhoseAppDoesNotStartIsNotASuccess(t *testing.T) {
	lateStartWatch = 0 // no watcher in this test
	defer func() { lateStartWatch = 30 * time.Minute }()
	m, _, rig, alarms := r1270Rig(t)
	_, err := m.RestoreFromRecoveryUnit("app")
	if !errors.Is(err, ErrRestoreNotStarted) {
		t.Fatalf("a restore whose app did not start must not be a success, got %v", err)
	}
	if want := util.MsgError("err.backup.restore_not_started", "app").Error(); err.Error() != want {
		t.Fatalf("the household reads the not-started sentence, got %q", err.Error())
	}
	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
		t.Fatalf("the restore's copy must be KEPT (not released) while the app has not started, found %v", left)
	}
	if len(*alarms) != 1 || !strings.Contains((*alarms)[0], "unit restore") {
		t.Fatalf("the operator gets exactly one alarm, got %v", *alarms)
	}
	if len(rig.poured) != 0 {
		t.Fatalf("a health timeout must NOT pour the old data back (a slow first start is common): %v", rig.poured)
	}
}

// Health passes → as before: success, the copy released.
func TestR1270_AHealthyRestoreIsASuccessAsBefore(t *testing.T) {
	m, p, rig, alarms := r1270Rig(t)
	p.up.Store(true)
	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
		t.Fatalf("a healthy restore is a success: %v", err)
	}
	if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 {
		t.Fatalf("the copy is released after a healthy restore, found %v", left)
	}
	if len(*alarms) != 0 {
		t.Fatalf("no alarm for a healthy restore: %v", *alarms)
	}
}

// A late start turns the record into a success and releases the kept copy (the normal rule).
func TestR1270_ALateStartTurnsTheRecordIntoASuccess(t *testing.T) {
	lateStartWatch, lateStartPoll = 2*time.Second, 5*time.Millisecond
	defer func() { lateStartWatch, lateStartPoll = 30*time.Minute, 30*time.Second }()
	m, p, rig, _ := r1270Rig(t)
	m.BeginRestoreOp("restore", "app")
	_, err := m.RestoreFromRecoveryUnit("app")
	if !errors.Is(err, ErrRestoreNotStarted) {
		t.Fatalf("precondition: not started, got %v", err)
	}
	m.EndRestoreOpNotStarted(err.Error())
	if st := m.RestoreStatus(); st.Last == nil || st.Last.OK || !st.Last.NotStarted {
		t.Fatalf("the record must say not ok / not started: %+v", st.Last)
	}
	p.up.Store(true)
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if st := m.RestoreStatus(); st.Last != nil && st.Last.OK {
			break
		}
		time.Sleep(5 * time.Millisecond)
	}
	st := m.RestoreStatus()
	if st.Last == nil || !st.Last.OK || st.Last.NotStarted {
		t.Fatalf("a late start must turn the record into a success: %+v", st.Last)
	}
	for i := 0; i < 100 && len(undoCopies(t, rig.undoRoot)) != 0; i++ {
		time.Sleep(5 * time.Millisecond)
	}
	if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
		t.Fatalf("the kept copy is released once the app runs, found %v", left)
	}
}

// The off-site restore: the same verdict, its undo copy kept.
func TestR1270_TheOffsiteRestoreToo(t *testing.T) {
	lateStartWatch = 0
	defer func() { lateStartWatch = 30 * time.Minute }()
	m, prov, _ := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
	m.stackProvider = &notRunningRecordingProvider{prov}
	_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
	if !errors.Is(err, ErrRestoreNotStarted) {
		t.Fatalf("an off-site restore whose app did not start must not be a success, got %v", err)
	}
}

type notRunningRecordingProvider struct{ *recordingProvider }

func (p *notRunningRecordingProvider) RefreshAndIsRunning(string) bool { return false }

// The record a late start may change is only that app's not-started record, and never while a restore runs.
func TestR1270_MarkLateStartTouchesOnlyItsOwnRecord(t *testing.T) {
	m := &Manager{}
	m.BeginRestoreOp("restore", "a")
	m.EndRestoreOp(false, "a failed")
	m.markLateStart("a", "started")
	if st := m.RestoreStatus(); st.Last.OK {
		t.Fatal("a plain failure must not become a success")
	}
	m.BeginRestoreOp("restore", "a")
	m.EndRestoreOpNotStarted("not yet")
	m.markLateStart("b", "started")
	if st := m.RestoreStatus(); st.Last.OK {
		t.Fatal("another app's start must not change this record")
	}
	m.BeginRestoreOp("restore", "a")
	m.markLateStart("a", "started")
	if st := m.RestoreStatus(); !st.Running {
		t.Fatal("a running restore must stay running")
	}
	_ = os.Getpid
}

// waitNoCopies waits (bounded) for a deferred release — fix A releases the copy after the stable window, not at return.
func waitNoCopies(t *testing.T, root string) []string {
	t.Helper()
	deadline := time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		if left := undoCopies(t, root); len(left) == 0 {
			return nil
		}
		time.Sleep(5 * time.Millisecond)
	}
	return undoCopies(t, root)
}