felhom-controller/controller/internal/backup/appstop_marker_test.go

package backup

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// R-166 part 2 — the app-stop crash marker.
//
// THE DISCIPLINE THAT MATTERS HERE (§10): a `defer` is not crash-safety, so a test that lets the
// deferred cleanup run proves nothing about a crash. Every "interrupted" test below simulates a
// SIGKILL by never reaching the restart — the marker is written, the process conceptually dies, and
// a FRESH guard over the SAME file does the recovering. That is exactly what Campaign 8 fault 10
// established on live hardware: a SIGKILL runs no deferred function, and what brought the stacks
// back was the marker read at startup.

type fakeStarter struct {
	starts   []string
	failWith map[string]error
}

func (f *fakeStarter) StartStack(name string) error {
	f.starts = append(f.starts, name)
	if err := f.failWith[name]; err != nil {
		return err
	}
	return nil
}

func newGuard(t *testing.T, dir string) (*AppStopGuard, *fakeStarter) {
	t.Helper()
	s := &fakeStarter{}
	g := NewAppStopGuard(filepath.Join(dir, "appstop-state.json"), log.New(io.Discard, "", 0))
	g.SetStarter(s)
	return g, s
}

func markerPath(dir string) string { return filepath.Join(dir, "appstop-state.json") }

func markerExists(t *testing.T, dir string) bool {
	t.Helper()
	_, err := os.Stat(markerPath(dir))
	if err != nil && !os.IsNotExist(err) {
		t.Fatal(err)
	}
	return err == nil
}

// --- Scenario E — a crash mid-backup brings the app back -----------------------------------------

func TestRecover_InterruptedVolumeDump_RestartsTheAppAndClearsTheMarker(t *testing.T) {
	dir := t.TempDir()

	// --- process 1: an operation stops the app and is KILLED. No End(), no defer, no cleanup. ---
	g1, _ := newGuard(t, dir)
	if err := g1.Begin("volume-dump:immich", ReasonVolumeDump, []string{"immich"}); err != nil {
		t.Fatalf("Begin: %v", err)
	}
	if !markerExists(t, dir) {
		t.Fatal("Begin did not write a marker — nothing would survive the kill")
	}
	// <SIGKILL here> — g1 is abandoned deliberately; nothing else is called on it.

	// --- process 2: a fresh controller starts and recovers from the file alone. ---
	g2, starter := newGuard(t, dir)
	res := g2.Recover()

	if len(starter.starts) != 1 || starter.starts[0] != "immich" {
		t.Fatalf("started %v, want exactly [immich] — the app was left stranded by the interrupted backup", starter.starts)
	}
	if res == nil || len(res.Restarted) != 1 || res.Restarted[0] != "immich" {
		t.Fatalf("recovery result = %+v, want immich restarted", res)
	}
	if res.Reason != ReasonVolumeDump {
		t.Fatalf("reason = %q, want %q — the operator must be told WHICH operation was interrupted", res.Reason, ReasonVolumeDump)
	}
	if markerExists(t, dir) {
		t.Fatal("the marker survived a successful recovery — the next boot would restart the app again")
	}
	// The operator-facing text must name the interruption, not merely report a restart.
	if msg := res.Message(); msg == "" || !strings.Contains(msg, "interrupted") {
		t.Fatalf("operator message %q does not say the operation was interrupted", msg)
	}
}

func TestRecover_NoMarker_IsASilentNoOp(t *testing.T) {
	dir := t.TempDir()
	g, starter := newGuard(t, dir)
	if res := g.Recover(); res != nil {
		t.Fatalf("Recover reported %+v on a box with no marker", res)
	}
	if len(starter.starts) != 0 {
		t.Fatalf("started %v with no marker present", starter.starts)
	}
}

func TestRecover_FailedRestart_KEEPSTheMarkerForTheNextStartup(t *testing.T) {
	// The single most important failure behaviour: clearing a marker whose restart failed would
	// erase the only durable record that an app is owed one. The app is genuinely still down.
	dir := t.TempDir()
	g1, _ := newGuard(t, dir)
	if err := g1.Begin("volume-dump:immich", ReasonVolumeDump, []string{"immich", "nextcloud"}); err != nil {
		t.Fatal(err)
	}

	g2, starter := newGuard(t, dir)
	starter.failWith = map[string]error{"immich": errors.New("compose up: no such image")}
	res := g2.Recover()

	if len(res.Failed) != 1 || res.Failed[0] != "immich" {
		t.Fatalf("failed=%v, want [immich]", res.Failed)
	}
	if len(res.Restarted) != 1 || res.Restarted[0] != "nextcloud" {
		t.Fatalf("restarted=%v, want [nextcloud] — one app failing must not abort the others", res.Restarted)
	}
	if !markerExists(t, dir) {
		t.Fatal("the marker was cleared even though a restart FAILED — the next startup would not retry")
	}
	if msg := res.Message(); !strings.Contains(msg, "NOT be restarted") {
		t.Fatalf("operator message %q does not report the failure", msg)
	}
	if d := res.Detail(); !strings.Contains(d, "restart_failed") || !strings.Contains(d, "immich") {
		t.Fatalf("detail %q does not name which app failed", d)
	}
}

func TestRecover_IsIdempotentAcrossRepeatedStartups(t *testing.T) {
	dir := t.TempDir()
	g1, _ := newGuard(t, dir)
	if err := g1.Begin("op", ReasonOffboxReconstitute, []string{"immich"}); err != nil {
		t.Fatal(err)
	}
	g2, s2 := newGuard(t, dir)
	g2.Recover()
	g3, s3 := newGuard(t, dir)
	g3.Recover()

	if len(s2.starts) != 1 {
		t.Fatalf("first recovery started %v", s2.starts)
	}
	if len(s3.starts) != 0 {
		t.Fatalf("a SECOND startup restarted %v again — the marker was not cleared", s3.starts)
	}
}

func TestRecover_CorruptMarkerIsQuarantinedNotSilentlySkipped(t *testing.T) {
	// §9.4: never a silent skip. A corrupt marker cannot be acted on, but it must leave a trace —
	// otherwise a genuinely interrupted operation vanishes without evidence.
	dir := t.TempDir()
	if err := os.WriteFile(markerPath(dir), []byte("{not json"), 0o600); err != nil {
		t.Fatal(err)
	}
	g, starter := newGuard(t, dir)
	if res := g.Recover(); res != nil {
		t.Fatalf("a corrupt marker produced a recovery result %+v", res)
	}
	if len(starter.starts) != 0 {
		t.Fatalf("apps were started from a corrupt marker: %v", starter.starts)
	}
	if markerExists(t, dir) {
		t.Fatal("the corrupt marker was left in place — it would be re-read forever")
	}
	quarantined, _ := filepath.Glob(markerPath(dir) + ".corrupt-*")
	if len(quarantined) != 1 {
		t.Fatalf("the corrupt marker was not quarantined (found %d) — it was silently dropped", len(quarantined))
	}
}

func TestRecover_NoStarterWiredKeepsTheMarker(t *testing.T) {
	// D-b's safety rule: never worse than not having the file. With no starter the guard cannot act,
	// so it must keep the record for a startup that can, rather than clear it and lose the app.
	dir := t.TempDir()
	g1, _ := newGuard(t, dir)
	if err := g1.Begin("op", ReasonVolumeDump, []string{"immich"}); err != nil {
		t.Fatal(err)
	}
	g2 := NewAppStopGuard(markerPath(dir), log.New(io.Discard, "", 0)) // deliberately no SetStarter
	if res := g2.Recover(); res != nil {
		t.Fatalf("recovered without a starter: %+v", res)
	}
	if !markerExists(t, dir) {
		t.Fatal("the marker was cleared with no starter wired — the app would never come back")
	}
}

func TestNilGuardIsInert(t *testing.T) {
	// A caller that was never wired must degrade to pre-v0.189.0 behaviour, not panic.
	var g *AppStopGuard
	if err := g.Begin("op", ReasonVolumeDump, []string{"x"}); err != nil {
		t.Fatalf("nil guard Begin returned %v", err)
	}
	g.End()
	if res := g.Recover(); res != nil {
		t.Fatalf("nil guard recovered %+v", res)
	}
}

func TestMarkerContentsAreDiagnosable(t *testing.T) {
	dir := t.TempDir()
	g, _ := newGuard(t, dir)
	if err := g.Begin("volume-dump:immich", ReasonVolumeDump, []string{"immich"}); err != nil {
		t.Fatal(err)
	}
	raw, err := os.ReadFile(markerPath(dir))
	if err != nil {
		t.Fatal(err)
	}
	var m AppStopMarker
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatalf("the marker on disk is not readable JSON: %v", err)
	}
	if !m.Active || m.OpID != "volume-dump:immich" || m.Reason != ReasonVolumeDump ||
		len(m.Stacks) != 1 || m.Stacks[0] != "immich" || m.StartedAt.IsZero() {
		t.Fatalf("the marker does not record enough to diagnose the interruption: %+v", m)
	}
	// 0600 — it names customer apps.
	fi, err := os.Stat(markerPath(dir))
	if err != nil {
		t.Fatal(err)
	}
	if fi.Mode().Perm() != 0o600 {
		t.Fatalf("marker mode = %v, want 0600", fi.Mode().Perm())
	}
}

func TestBeginWithNoStacksWritesNothing(t *testing.T) {
	dir := t.TempDir()
	g, _ := newGuard(t, dir)
	if err := g.Begin("op", ReasonVolumeDump, nil); err != nil {
		t.Fatal(err)
	}
	if markerExists(t, dir) {
		t.Fatal("a marker was written for an operation that stops nothing")
	}
}

// --- Scenarios E/F — DumpAppVolumesSafe, the primary site ----------------------------------------

// inspectingProvider is the StackDataProvider slice DumpAppVolumesSafe touches. It records whether
// the marker file EXISTED at each step — the positive observable for the ordering property. An
// absent log line is not evidence (standing rule 3); the file's presence at the moment of the stop
// is.
//
// GetDockerVolumes returns nothing, so the dump itself is a no-op and no Docker is involved — the
// stop/start bracket around it is what is under test.
type inspectingProvider struct {
	StackDataProvider
	markerFile          string
	events              []string
	stopErr             error
	startErr            error
	markerPresentAtStop bool
	markerAtStartCall   bool
	// panicOnVolumes simulates a hard abort (SIGKILL/power cut) at the point the dump begins: the
	// unwind skips the restart statement, exactly as a kill would.
	panicOnVolumes bool
}

func (p *inspectingProvider) GetDockerVolumes(string) []string {
	if p.panicOnVolumes {
		panic("simulated hard abort mid-dump")
	}
	return nil
}

func (p *inspectingProvider) StopStack(name string) error {
	_, err := os.Stat(p.markerFile)
	p.markerPresentAtStop = err == nil
	p.events = append(p.events, "stop:"+name)
	return p.stopErr
}

func (p *inspectingProvider) StartStack(name string) error {
	_, err := os.Stat(p.markerFile)
	p.markerAtStartCall = err == nil
	p.events = append(p.events, "start:"+name)
	return p.startErr
}

func newDumpManager(t *testing.T, dir string, p *inspectingProvider) *Manager {
	t.Helper()
	lg := log.New(io.Discard, "", 0)
	m := &Manager{logger: lg, stackProvider: p, systemDataPath: dir}
	m.appStop = NewAppStopGuard(markerPath(dir), lg)
	return m
}

func TestDumpAppVolumesSafe_MarkerCoversTheWholeStopStartWindow(t *testing.T) {
	// Scenario F, the happy path: the marker is on disk BEFORE the stop, still on disk for the whole
	// time the app is down, and GONE once the restart succeeds.
	dir := t.TempDir()
	p := &inspectingProvider{markerFile: markerPath(dir)}
	m := newDumpManager(t, dir, p)

	if err := m.DumpAppVolumesSafe("immich"); err != nil {
		t.Fatalf("DumpAppVolumesSafe: %v", err)
	}

	if !p.markerPresentAtStop {
		t.Fatal("the marker was NOT on disk when the app was stopped — a crash one instruction later " +
			"strands the app, which is the entire failure this marker exists to prevent")
	}
	if !p.markerAtStartCall {
		t.Fatal("the marker was already gone while the app was still down")
	}
	if markerExists(t, dir) {
		t.Fatal("the marker survived a dump whose restart succeeded — the next boot would restart the app again")
	}
	if len(p.events) != 2 || p.events[0] != "stop:immich" || p.events[1] != "start:immich" {
		t.Fatalf("events=%v, want [stop:immich start:immich]", p.events)
	}
}

func TestDumpAppVolumesSafe_Interrupted_RecoveryBringsTheAppBack(t *testing.T) {
	// Scenario E end-to-end THROUGH THE PRODUCTION PATH, and WITHOUT running any cleanup.
	//
	// The abort is real: GetDockerVolumes panics, which unwinds out of DumpAppVolumesSafe AFTER the
	// marker was written and the app stopped, and BEFORE the restart statement — and because that
	// restart is a plain statement, not a defer, it never runs. That is the shape of a hard kill.
	//
	// The earlier version of this test called m.appStop.Begin itself, which meant it proved the
	// marker type worked and NOT that DumpAppVolumesSafe uses it — it survived the red-proof that
	// deleted the production Begin call. Driving the real function is what makes the proof bite.
	//
	// RED-PROOF: delete the `m.appStop.Begin(...)` call from DumpAppVolumesSafe and this test fails —
	// nothing is written, so nothing is recovered. Demonstrated in REPORT.md §5.
	dir := t.TempDir()
	p := &inspectingProvider{markerFile: markerPath(dir), panicOnVolumes: true}
	m := newDumpManager(t, dir, p)

	func() {
		defer func() {
			if recover() == nil {
				t.Error("the simulated abort did not fire — this test proves nothing")
			}
		}()
		_ = m.DumpAppVolumesSafe("immich")
	}()

	if !p.markerPresentAtStop {
		t.Fatal("the app was stopped before any marker existed")
	}
	if p.markerAtStartCall {
		t.Fatal("the restart ran despite the abort — the simulation is wrong, not the code")
	}

	// <the controller is gone> — a fresh one starts and recovers from the file alone.
	g, starter := newGuard(t, dir)
	res := g.Recover()

	if len(starter.starts) != 1 || starter.starts[0] != "immich" {
		t.Fatalf("started %v — the app stopped by the interrupted dump was not brought back", starter.starts)
	}
	if res == nil || res.Reason != ReasonVolumeDump {
		t.Fatalf("recovery did not name the volume dump as the interrupted operation: %+v", res)
	}
	if markerExists(t, dir) {
		t.Fatal("the marker was not cleared after a successful recovery")
	}
}

func TestDumpAppVolumesSafe_FailedRestartKeepsTheMarker(t *testing.T) {
	dir := t.TempDir()
	p := &inspectingProvider{markerFile: markerPath(dir), startErr: errors.New("compose up failed")}
	m := newDumpManager(t, dir, p)

	if err := m.DumpAppVolumesSafe("immich"); err == nil {
		t.Fatal("a failed restart must surface as an error")
	}
	if !markerExists(t, dir) {
		t.Fatal("the marker was cleared even though the restart FAILED — the app is still down and " +
			"nothing records that it is owed a restart")
	}
}

func TestDumpAppVolumesSafe_FailedStopClearsTheMarker(t *testing.T) {
	// Nothing was stopped, so nothing is owed a restart. A stranded marker here would cost a
	// spurious restart at the next startup AND a false "a backup was interrupted" alert.
	dir := t.TempDir()
	p := &inspectingProvider{markerFile: markerPath(dir), stopErr: errors.New("stack is protected")}
	m := newDumpManager(t, dir, p)

	if err := m.DumpAppVolumesSafe("traefik"); err == nil {
		t.Fatal("a failed stop must surface as an error")
	}
	if markerExists(t, dir) {
		t.Fatal("a marker was left behind for an app that was never stopped")
	}
}