felhom-controller/controller/internal/quiesce/notify_test.go

package quiesce

import (
	"context"
	"testing"
	"time"
)

// R-97a/b — a failing backup must be HEARD, and must not blame the apps.
//
// Both halves are inert-seam risks, so nothing here asserts "no error occurred". The failure mode
// this suite exists to catch is a route that silently does not exist — which is exactly what an
// empty event list looks like.

// recEvent is one captured hub event.
type recEvent struct {
	kind string // "failed" | "recovered"
	tier string
	msg  string
	err  string
}

// recNotifier captures what would reach the hub.
type recNotifier struct{ events []recEvent }

func (r *recNotifier) BackupFailed(tier, message, errMsg string) {
	r.events = append(r.events, recEvent{"failed", tier, message, errMsg})
}
func (r *recNotifier) BackupRecovered(tier, message string) {
	r.events = append(r.events, recEvent{"recovered", tier, message, ""})
}
func (r *recNotifier) count(kind, tier string) int {
	n := 0
	for _, e := range r.events {
		if e.kind == kind && (tier == "" || e.tier == tier) {
			n++
		}
	}
	return n
}

// ── SCENARIO A — heard ONCE, not once per retry ──────────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): drop the `n == 1` guard in noteTierFailure so every failure emits,
// then step past each backoff — this fails with
//
//	"a failing tier must be reported ONCE per run of failures, got 4"
//
// Restored.
func TestNotify_FailingTierIsReportedOncePerRun(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)
	rec := &recNotifier{}
	l.SetTierNotifier(rec)

	for i := 0; i < 4; i++ { // four ATTEMPTS, each past the previous backoff
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
		now = now.Add(backoffFor(i+1) + time.Minute)
	}
	if got := rec.count("failed", "felhom-pbs"); got != 1 {
		t.Fatalf("a failing tier must be reported ONCE per run of failures, got %d", got)
	}
	if rec.events[0].tier != "felhom-pbs" {
		t.Fatalf("the TIER must be named — 'a backup failed' without saying which tier is not actionable; got %q", rec.events[0].tier)
	}
	if rec.events[0].err == "" {
		t.Fatal("the underlying error must travel with the event, or the operator has to go digging")
	}
}

// ── SCENARIO B — recovery is announced ───────────────────────────────────────────────────────
func TestNotify_RecoveryIsAnnounced(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)
	rec := &recNotifier{}
	l.SetTierNotifier(rec)

	if err := l.runOnce(context.Background()); err != nil { // fail → reported
		t.Fatal(err)
	}
	be.mu.Lock()
	be.phases["felhom-pbs"] = []string{phaseDone}
	be.phaseIdx["felhom-pbs"] = 0
	be.mu.Unlock()
	now = now.Add(backoffFor(1) + time.Minute)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if got := rec.count("recovered", "felhom-pbs"); got != 1 {
		t.Fatalf("an operator told a tier BROKE must be told it HEALED; recovery events = %d", got)
	}
}

// A tier that never failed must not emit a recovery on every healthy backup.
func TestNotify_HealthyTierIsSilent(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("nothing-fails", "local")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)
	rec := &recNotifier{}
	l.SetTierNotifier(rec)

	for i := 0; i < 3; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
		be.mu.Lock()
		be.phaseIdx["local"] = 0
		be.mu.Unlock()
		now = now.Add(5 * time.Minute)
	}
	if len(rec.events) != 0 {
		t.Fatalf("a healthy tier must emit NOTHING; got %+v", rec.events)
	}
}

// ── SCENARIO C — one failing tier does not mask another ──────────────────────────────────────
//
// The controller half: BOTH tiers must emit, each naming itself. (The hub half — the operator
// cooldown keyed customerID:eventType, which would swallow the second tier's EMAIL within the hour —
// is fixed hub-side by cooldownTierSuffix; see REPORT.md.)
//
// COMPANION RED-PROOF (observed): key the breaker on a constant instead of t.target (a single global
// failure record) and this fails with
//
//	"both tiers must be reported; got local=1 felhom-pbs=0"
//
// Restored.
func TestNotify_BothFailingTiersAreReported(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := newTierBackend()
	for _, tr := range []string{"local", "felhom-pbs"} {
		be.tiers = append(be.tiers, BackupTier{Target: tr})
		be.dueSet[tr] = true
		be.phases[tr] = []string{phaseFailed}
	}
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)
	rec := &recNotifier{}
	l.SetTierNotifier(rec)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	gotLocal, gotPBS := rec.count("failed", "local"), rec.count("failed", "felhom-pbs")
	if gotLocal != 1 || gotPBS != 1 {
		t.Fatalf("both tiers must be reported; got local=%d felhom-pbs=%d — one broken tier must not mask another",
			gotLocal, gotPBS)
	}
}

// A nil notifier (unprovisioned guest) must not panic.
func TestNotify_NilNotifierIsSafe(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now) // no SetTierNotifier
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("a loop with no notifier must still back up: %v", err)
	}
}

// ── SCENARIO E — quiesce does not blame the apps ─────────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): make SuppressedStacks return nil unconditionally (the pre-fix
// shape) and this fails with
//
//	"R-97b: the apps we stopped for a backup must not be reported down; bookstack still alarmed"
//
// Restored.
func TestSuppress_StacksWeStoppedAreNotReportedDown(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack", "immich"}}
	l := breakerLoop(t, be, st, &now)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	// Immediately after the cycle: both stacks are inside the grace window.
	sup := l.SuppressedStacks()
	for _, name := range []string{"bookstack", "immich"} {
		if !sup[name] {
			t.Fatalf("R-97b: the apps we stopped for a backup must not be reported down; %s still alarmed", name)
		}
	}
	// A stack the cycle never touched is NOT suppressed — the window is scoped, not blanket.
	if sup["vaultwarden"] {
		t.Fatal("suppression must cover only the stacks THIS cycle stopped, not every app")
	}
}

// ── SCENARIO F — an app that really fails to restart STILL alarms ────────────────────────────
//
// F is what makes E safe: a suite containing only E would pass against permanent suppression, which
// turns a loud false alarm into a silent real one — the R-88 Scenario D trap in a new costume.
//
// COMPANION RED-PROOF (observed): drop the expiry check in SuppressedStacks (return every entry
// regardless of `until`) and this fails with
//
//	"CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed 181s after the
//	 cycle; permanent suppression is a silent real alarm"
//
// Restored.
func TestSuppress_ContractGraceWindowExpires(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if !l.SuppressedStacks()["bookstack"] {
		t.Fatal("setup: bookstack should be suppressed immediately after the cycle")
	}
	// Just inside the window — still suppressed.
	now = now.Add(quiesceAlarmGrace - time.Second)
	if !l.SuppressedStacks()["bookstack"] {
		t.Fatalf("suppression must hold for the full grace window (%s) — an app needs time to come up", quiesceAlarmGrace)
	}
	// Past the window — it MUST alarm again.
	now = now.Add(2 * time.Second)
	if l.SuppressedStacks()["bookstack"] {
		t.Fatalf("CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed %s after the cycle; "+
			"permanent suppression is a silent real alarm", quiesceAlarmGrace+time.Second)
	}
}

// A long-running cycle must keep suppressing throughout — a first full offsite snapshot legitimately
// runs for hours, and the app is legitimately down that whole time.
func TestSuppress_HoldsForTheWholeCycleHowLongItRuns(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	l := breakerLoop(t, newTierBackend(), &fakeStacks{}, &now)
	l.markQuiesced([]string{"immich"})
	now = now.Add(6 * time.Hour) // still quiesced; no unquiesce yet
	if !l.SuppressedStacks()["immich"] {
		t.Fatal("an app stopped by a still-running cycle must stay suppressed, however long the backup takes")
	}
	l.markUnquiesced([]string{"immich"})
	now = now.Add(quiesceAlarmGrace + time.Second)
	if l.SuppressedStacks()["immich"] {
		t.Fatal("once unquiesced, the grace window must still expire")
	}
}