felhom-controller/controller/internal/backup/appstop_suppress_test.go

package backup

import (
	"errors"
	"io"
	"log"
	"testing"
	"time"
)

// ── R-330 — the nightly volume dump must not alarm about the app it is holding down ──────────────
//
// The defect these pin, measured on demo-hp 2026-08-30 with controller 0.223.0: `DumpAppVolumesSafe`
// stopped a stack for ~13 s to tar its volumes while the `deadapp-check` job ran every 30 s, so the
// scan caught the stack mid-cycle and pushed `app_start_failed` to the customer. 61 e-mails.
//
// The tests are split by what they must not lose:
//   - the SUPPRESSION exists and covers the whole window (the fix), and
//   - it can NEVER latch (the thing the fix must not break) — a restart that failed, a hold nothing
//     released, and a stranded set from an earlier operation each let the alarm through.
//
// RED-PROOF (run 2026-08-30, recorded in REPORT.md): removing the `g.markStopped(stackNames)` call
// from `AppStopGuard.Begin` fails TestVolumeDump_SuppressesTheAlarmForTheAppItIsHolding with
// `suppressed at stop = map[]` — the exact pre-fix shape that produced the e-mails.

// suppressWatchProvider drives the REAL DumpAppVolumesSafe and records the alarm-suppression set at
// the two moments that matter: while the app is stopped, and at the restart call. Asserting on the
// suppression set (what the dead-app scanner actually reads) rather than on a log line is the
// standing-rule-3 positive observable — and it is the CONSEQUENCE, not the mechanism.
type suppressWatchProvider struct {
	StackDataProvider
	guard *AppStopGuard

	suppressedAtStop  map[string]bool
	suppressedAtStart map[string]bool
	startErr          error
}

func (p *suppressWatchProvider) GetDockerVolumes(string) []string { return nil }

func (p *suppressWatchProvider) StopStack(string) error {
	p.suppressedAtStop = p.guard.SuppressedStacks()
	return nil
}

func (p *suppressWatchProvider) StartStack(string) error {
	p.suppressedAtStart = p.guard.SuppressedStacks()
	return p.startErr
}

// newSuppressManager builds a Manager over a real guard and returns both.
func newSuppressManager(t *testing.T, p *suppressWatchProvider) *Manager {
	t.Helper()
	dir := t.TempDir()
	lg := log.New(io.Discard, "", 0)
	m := &Manager{logger: lg, stackProvider: p, systemDataPath: dir}
	m.appStop = NewAppStopGuard(markerPath(dir), lg)
	p.guard = m.appStop
	return m
}

func TestVolumeDump_SuppressesTheAlarmForTheAppItIsHolding(t *testing.T) {
	// THE FIX. Through the production path, not by calling Begin from the test: an earlier sibling
	// test in this package was rewritten for exactly that reason — proving the guard works is not
	// proving DumpAppVolumesSafe uses it.
	p := &suppressWatchProvider{}
	m := newSuppressManager(t, p)

	if err := m.DumpAppVolumesSafe("bookstack"); err != nil {
		t.Fatalf("DumpAppVolumesSafe: %v", err)
	}

	if !p.suppressedAtStop["bookstack"] {
		t.Fatalf("suppressed at stop = %v, want bookstack — the dead-app scan runs every 30s and the "+
			"stack is down for ~13s, so an unsuppressed window is the false `app_start_failed` e-mail "+
			"the customer received nightly", p.suppressedAtStop)
	}
	if !p.suppressedAtStart["bookstack"] {
		t.Fatalf("suppressed at restart = %v, want bookstack — the app is STILL down at this point",
			p.suppressedAtStart)
	}
	// And it is still suppressed after the restart returns: R-97b's BookStack alarmed while `starting`
	// / `unhealthy`, which is neither stopped nor healthy, so a window that ends at the restart CALL
	// closes too early to fix anything.
	if !m.appStop.SuppressedStacks()["bookstack"] {
		t.Fatal("the suppression ended the instant the restart returned — an app that is up but not " +
			"yet healthy still reads as down, which is the exact shape R-97b was filed for")
	}
}

func TestVolumeDump_SuppressionExpiresSoARealOutageStillAlarms(t *testing.T) {
	// The window is a BOUNDED DELAY in reporting a real failure, never its loss. Without this the fix
	// would trade a loud false alarm for a silent real one — F-CRIT-1 and R-88 Scenario D.
	p := &suppressWatchProvider{}
	m := newSuppressManager(t, p)

	base := time.Now()
	m.appStop.now = func() time.Time { return base }

	if err := m.DumpAppVolumesSafe("bookstack"); err != nil {
		t.Fatalf("DumpAppVolumesSafe: %v", err)
	}
	if !m.appStop.SuppressedStacks()["bookstack"] {
		t.Fatal("not suppressed immediately after the restart")
	}

	// One second before the grace closes: still suppressed.
	m.appStop.now = func() time.Time { return base.Add(appStopAlarmGrace - time.Second) }
	if !m.appStop.SuppressedStacks()["bookstack"] {
		t.Fatalf("the grace closed early — a stack restarted %s ago must still be exempt", appStopAlarmGrace-time.Second)
	}

	// At the grace boundary: the alarm owns it again.
	m.appStop.now = func() time.Time { return base.Add(appStopAlarmGrace) }
	if m.appStop.SuppressedStacks()["bookstack"] {
		t.Fatalf("still suppressed %s after the restart — an app that genuinely failed to come back "+
			"would never be reported", appStopAlarmGrace)
	}
}

func TestVolumeDump_FailedRestartAlarmsImmediately(t *testing.T) {
	// The primary anti-latch mechanism. We stopped it, we could not give it back: it is genuinely
	// down, and it must alarm on the NEXT scan — not after any grace at all.
	p := &suppressWatchProvider{startErr: errors.New("compose up failed")}
	m := newSuppressManager(t, p)

	if err := m.DumpAppVolumesSafe("bookstack"); err == nil {
		t.Fatal("a failed restart must surface as an error")
	}

	if got := m.appStop.SuppressedStacks(); got["bookstack"] {
		t.Fatalf("suppressed = %v after a restart that FAILED — the app is down and the alarm is the "+
			"only thing that would tell anyone, which is F-CRIT-1 exactly", got)
	}
	// The durable marker is a SEPARATE concern and must be untouched by the release: it is what the
	// next startup retries from. Losing it here would trade a false alarm for a lost recovery.
	if !markerExists(t, m.systemDataPath) {
		t.Fatal("ReleaseFailed also cleared the crash marker — the next startup would not retry the restart")
	}
}

func TestSuppression_CannotOutliveTheBackstop(t *testing.T) {
	// The belt-and-braces for a hold nothing ever released. `End()` runs only on a restart that
	// SUCCEEDED, so a future failure path that forgets ReleaseFailed would otherwise silence an app
	// forever. Exceeding the backstop means something is wrong, and the right answer when something
	// is wrong is to let the alarm through.
	base := time.Now()
	g := NewAppStopGuard(markerPath(t.TempDir()), log.New(io.Discard, "", 0))
	g.now = func() time.Time { return base }

	if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err != nil {
		t.Fatalf("Begin: %v", err)
	}
	if !g.SuppressedStacks()["romm"] {
		t.Fatal("not suppressed while held")
	}

	g.now = func() time.Time { return base.Add(appStopMaxHold - time.Minute) }
	if !g.SuppressedStacks()["romm"] {
		t.Fatalf("the backstop fired early — a legitimate long operation would start alarming mid-run")
	}

	g.now = func() time.Time { return base.Add(appStopMaxHold) }
	if g.SuppressedStacks()["romm"] {
		t.Fatalf("a hold open-ended for %s is still suppressing the alarm — nothing released it and "+
			"nothing ever will", appStopMaxHold)
	}
}

func TestBeginReplacesThePreviousOperationsSet(t *testing.T) {
	// The marker file holds ONE operation, so a new Begin proves the previous one is over. Without
	// this, a set stranded by an operation that died between Begin and End would suppress its stacks
	// for the whole appStopMaxHold, even though a later operation has since taken over.
	g := NewAppStopGuard(markerPath(t.TempDir()), log.New(io.Discard, "", 0))

	if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err != nil {
		t.Fatalf("Begin romm: %v", err)
	}
	if err := g.Begin("volume-dump:kimai", ReasonVolumeDump, []string{"kimai"}); err != nil {
		t.Fatalf("Begin kimai: %v", err)
	}

	got := g.SuppressedStacks()
	if got["romm"] {
		t.Fatalf("suppressed = %v — romm's hold survived into a later operation that is not holding it", got)
	}
	if !got["kimai"] {
		t.Fatalf("suppressed = %v, want kimai — the current operation's own stack is not exempt", got)
	}
}

func TestSuppression_FailedBeginSuppressesNothing(t *testing.T) {
	// A Begin whose write failed means the caller REFUSES to stop the app (DumpAppVolumesSafe returns
	// early). Nothing is stopped, so nothing may be exempt — suppressing here would hide a genuinely
	// dead app that this operation never touched.
	g := NewAppStopGuard("/proc/felhom-nonexistent-dir/appstop.json", log.New(io.Discard, "", 0))

	if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err == nil {
		t.Skip("the unwritable path became writable — this environment cannot exercise the failure")
	}
	if got := g.SuppressedStacks(); got["romm"] {
		t.Fatalf("suppressed = %v after a Begin that FAILED — the app was never stopped", got)
	}
}

func TestNilGuardSuppressesNothing(t *testing.T) {
	// An unprovisioned guest has no guard. Suppressing nothing is the correct default, and nil-safety
	// is what lets the caller in main.go stay branch-free.
	var g *AppStopGuard
	if got := g.SuppressedStacks(); len(got) != 0 {
		t.Fatalf("a nil guard suppressed %v", got)
	}
	g.ReleaseFailed("romm") // must not panic
}