felhom-controller/controller/internal/quiesce/breaker_test.go

package quiesce

import (
	"context"
	"log"
	"strings"
	"testing"
	"time"
)

// R-88 — the failure breaker.
//
// THE ASSERTION THAT MATTERS IS A COUNT. The harm this fixes is not "a backup failed" and not "an
// error was logged" — it is the number of times the customer's apps were STOPPED AND RESTARTED for a
// backup that could not succeed. A test that asserts an error was logged passes against the exact
// pre-fix code, so every scenario below counts stop/start pairs instead.

// breakerLoop builds a Loop with a controllable clock and the window gate DISABLED, so these tests
// isolate the breaker from the window logic (which has its own suite).
func breakerLoop(t *testing.T, be Backend, st Stacks, now *time.Time) *Loop {
	t.Helper()
	l := testLoop(t, be, st)
	l.now = func() time.Time { return *now }
	return l
}

// tierBackendFailing builds a two-tier fake where `failing` always reports a failed backup.
func tierBackendFailing(failing string, tiers ...string) *tierBackend {
	b := newTierBackend()
	for _, tr := range tiers {
		b.tiers = append(b.tiers, BackupTier{Target: tr})
		b.dueSet[tr] = true
		if tr == failing {
			b.phases[tr] = []string{phaseFailed}
		} else {
			b.phases[tr] = []string{phaseDone}
		}
	}
	return b
}

// ── SCENARIO A — a failing backup stops re-quiescing ─────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): comment out the `dropBackedOffTiers` call in runOnce (the pre-fix
// shape — every due tier quiesces on every tick) and this fails with:
//
//	"R-88: 6 failing ticks stopped the apps 6 time(s); want 1 — the breaker did not defer anything"
//
// Restored.
func TestBreaker_FailingTierStopsRequiescing(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack", "immich"}}
	l := breakerLoop(t, be, st, &now)

	// Three ticks five minutes apart — EXACTLY the 2026-07-27 incident shape (09:02:57, 09:07:58,
	// 09:12:57), which produced three stop/start pairs. All three fall inside the first 15m backoff,
	// so a working breaker yields one.
	const ticks = 3
	for i := 0; i < ticks; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
		now = now.Add(5 * time.Minute)
	}

	const stacksPerQuiesce = 2 // bookstack + immich
	pairs := len(st.stoppedNames()) / stacksPerQuiesce
	if pairs != 1 {
		t.Fatalf("R-88: %d failing ticks stopped the apps %d time(s); want 1 — the breaker did not defer anything",
			ticks, pairs)
	}
	if got := len(st.startedNames()) / stacksPerQuiesce; got != pairs {
		t.Fatalf("every quiesce must unquiesce: %d stop(s) vs %d start(s)", pairs, got)
	}
	if got := l.breaker.failuresFor("felhom-pbs"); got != 1 {
		t.Fatalf("only the FIRST tick should have attempted (and failed); consecutive failures = %d, want 1", got)
	}
}

// The backoff must EXPIRE — a breaker that latches open is a silent backup outage, which is worse
// than the loop it replaces.
func TestBreaker_RetriesAfterTheBackoffExpires(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)

	if err := l.runOnce(context.Background()); err != nil { // attempt 1 → fails, arms 15m
		t.Fatal(err)
	}
	now = now.Add(breakerBaseDelay + time.Minute) // past the first backoff
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if got := len(st.stoppedNames()); got != 2 {
		t.Fatalf("after the backoff expired the tier must be retried: %d stop(s), want 2 — a permanent breaker is a silent outage", got)
	}
	if got := l.breaker.failuresFor("felhom-pbs"); got != 2 {
		t.Fatalf("the retry also failed, so the count must climb: got %d, want 2", got)
	}
}

// ── SCENARIO B — backoff resets on success ───────────────────────────────────────────────────
func TestBreaker_SuccessClearsTheBackoff(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)

	for i := 0; i < 3; i++ { // three failures, each past the previous backoff
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
		now = now.Add(backoffFor(i+1) + time.Minute)
	}
	if l.breaker.failuresFor("felhom-pbs") != 3 {
		t.Fatalf("setup: want 3 consecutive failures, got %d", l.breaker.failuresFor("felhom-pbs"))
	}

	// The tier recovers.
	be.mu.Lock()
	be.phases["felhom-pbs"] = []string{phaseDone}
	be.phaseIdx["felhom-pbs"] = 0
	be.mu.Unlock()
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if got := l.breaker.failuresFor("felhom-pbs"); got != 0 {
		t.Fatalf("a success must clear the backoff entirely; consecutive failures = %d, want 0", got)
	}

	// And the very NEXT cycle must run — no lingering penalty.
	before := len(st.stoppedNames())
	be.mu.Lock()
	be.phaseIdx["felhom-pbs"] = 0
	be.mu.Unlock()
	now = now.Add(5 * time.Minute)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if len(st.stoppedNames()) == before {
		t.Fatal("after a success the next cycle must run immediately — a recovered box must carry no penalty")
	}
}

// ── SCENARIO F — one failing tier does not suppress the other ────────────────────────────────
//
// COMPANION RED-PROOF (observed): make dropBackedOffTiers return nil when ANY tier is blocked (the
// tempting "global breaker" shape) and this fails with:
//
//	"R-88 Scenario F: the healthy local tier was backed up 1 time(s) across 4 ticks; want >= 3"
//
// Restored.
func TestBreaker_OneFailingTierDoesNotSuppressAHealthyOne(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "local", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)

	const ticks = 4
	for i := 0; i < ticks; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
		now = now.Add(5 * time.Minute)
	}

	localRuns := 0
	be.mu.Lock()
	for _, tr := range be.started {
		if tr == "local" {
			localRuns++
		}
	}
	be.mu.Unlock()
	if localRuns < 3 {
		t.Fatalf("R-88 Scenario F: the healthy local tier was backed up %d time(s) across %d ticks; want >= 3 — a broken tier must not halt a working one",
			localRuns, ticks)
	}
	if l.breaker.failuresFor("local") != 0 {
		t.Fatalf("the healthy tier must carry no failures; got %d", l.breaker.failuresFor("local"))
	}
}

// ── SCENARIO E — a manual trigger is never gated ─────────────────────────────────────────────
//
// TriggerNow bypasses due-ness and the window gate; it must bypass the breaker too. A human pressing
// „Mentés most" has made an explicit decision and must not be deferred by a scheduler's safety net.
func TestBreaker_ManualTriggerIsNeverGated(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	l := breakerLoop(t, be, st, &now)

	if err := l.runOnce(context.Background()); err != nil { // arm a backoff
		t.Fatal(err)
	}
	if _, blocked := l.breaker.blocked("felhom-pbs", now); !blocked {
		t.Fatal("setup: the tier should be in backoff")
	}
	before := len(st.stoppedNames())

	if err := l.TriggerNow(); err != nil {
		t.Fatalf("TriggerNow: %v", err)
	}
	deadline := time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		if len(st.stoppedNames()) > before {
			return // ran despite the active backoff — correct
		}
		time.Sleep(10 * time.Millisecond)
	}
	t.Fatal("a manual backup was deferred by the R-88 breaker; the human's explicit action must always run")
}

// ── The backoff schedule is a contract, not an emergent property ─────────────────────────────
func TestBreaker_BackoffSchedule(t *testing.T) {
	want := []time.Duration{
		15 * time.Minute, 30 * time.Minute, time.Hour, 2 * time.Hour, 4 * time.Hour,
		4 * time.Hour, 4 * time.Hour, // capped, and it KEEPS retrying — never permanent
	}
	for i, w := range want {
		if got := backoffFor(i + 1); got != w {
			t.Errorf("backoffFor(%d) = %s, want %s", i+1, got, w)
		}
	}
	if got := backoffFor(50); got != breakerMaxDelay {
		t.Fatalf("a long-broken tier must stay at the cap and keep retrying, got %s", got)
	}
	if backoffFor(0) != 0 {
		t.Fatal("no failures means no backoff")
	}
}

// The deferral is announced ONCE, when armed — not on every skipped tick.
func TestBreaker_LogsTheDeferralOncePerBackoff(t *testing.T) {
	now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
	be := tierBackendFailing("felhom-pbs", "felhom-pbs")
	st := &fakeStacks{running: []string{"bookstack"}}
	var buf strings.Builder
	l := breakerLoop(t, be, st, &now)
	l.logger = log.New(&buf, "", 0)

	for i := 0; i < 5; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
		now = now.Add(5 * time.Minute)
	}
	if n := strings.Count(buf.String(), "failed 1 time(s) in a row"); n != 1 {
		t.Fatalf("the deferral must be logged ONCE per backoff period, got %d — a 5-minute loop logging every tick buries the signal", n)
	}
}