felhom-controller/controller/internal/quiesce/tiers_test.go

package quiesce

import (
	"context"
	"fmt"
	"io"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// R-82 Slice B — one quiesce window, two tiers.
//
// Two properties are load-bearing and neither is provable by "no error was returned":
//   1. On the both-due night there is EXACTLY ONE stop/start pair. Two would mean two app outages
//      for one night's work, undoing the whole argument for weekly-over-daily.
//   2. A new controller against an OLD agent still TAKES A BACKUP. The hollow version of that test
//      asserts "no error" while silently skipping the backup — the exact failure it exists to catch.

// ---- fakes -------------------------------------------------------------------------------

// NOTE: fakeStacks comes from quiesce_test.go (same package) — reused rather than duplicated.
// Counts come from len(stoppedNames()) / len(startedNames()).

// tierBackend is a multi-tier fake agent. phases[target] is the phase sequence returned by
// successive BackupStatusFor calls for that tier.
type tierBackend struct {
	mu           sync.Mutex
	tiers        []BackupTier
	tiersErr     error
	dueSet       map[string]bool
	phases       map[string][]string
	phaseIdx     map[string]int
	started      []string // targets StartBackupFor/StartBackup was called with, in order
	untargetedDue bool
	startErrOn   string
	// stacks (optional) lets a start sample how many restarts have happened SO FAR — the direct
	// way to assert "the app had not resumed when this tier started".
	stacks        *fakeStacks
	startsAtStart []int
}

func newTierBackend() *tierBackend {
	return &tierBackend{
		dueSet: map[string]bool{}, phases: map[string][]string{}, phaseIdx: map[string]int{},
	}
}

func (b *tierBackend) Tiers(context.Context) ([]BackupTier, error) {
	if b.tiersErr != nil {
		return nil, b.tiersErr
	}
	return b.tiers, nil
}
func (b *tierBackend) DueFor(_ context.Context, target string) (bool, *int64, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.dueSet[target], nil, nil
}
func (b *tierBackend) StartBackupFor(_ context.Context, target string) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.startErrOn == target {
		return "", fmt.Errorf("simulated start failure on %s", target)
	}
	b.started = append(b.started, target)
	if b.stacks != nil {
		b.startsAtStart = append(b.startsAtStart, len(b.stacks.startedNames()))
	}
	return "job-" + target, nil
}
func (b *tierBackend) BackupStatusFor(_ context.Context, target string) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	seq := b.phases[target]
	i := b.phaseIdx[target]
	if i < len(seq) {
		b.phaseIdx[target]++
		return seq[i], nil
	}
	if len(seq) > 0 {
		return seq[len(seq)-1], nil
	}
	return phaseDone, nil
}

// The untargeted (pre-R-82) surface.
func (b *tierBackend) Due(context.Context) (bool, *int64, error) {
	return b.untargetedDue, nil, nil
}
func (b *tierBackend) StartBackup(context.Context) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.started = append(b.started, "(untargeted)")
	return "job-untargeted", nil
}
func (b *tierBackend) BackupStatus(context.Context) (string, error) { return phaseDone, nil }

func (b *tierBackend) restartsWhenEachTierStarted() []int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]int(nil), b.startsAtStart...)
}

func (b *tierBackend) startedTargets() []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]string(nil), b.started...)
}

func newTierLoop(t *testing.T, be Backend, st *fakeStacks, logTo io.Writer) *Loop {
	t.Helper()
	if logTo == nil {
		logTo = io.Discard
	}
	return New(Options{
		Backend:    be,
		Stacks:     st,
		MarkerPath: filepath.Join(t.TempDir(), "quiesce-state.json"),
		StatusPoll: time.Millisecond,
		MaxQuiesce: 30 * time.Second,
		Logger:     log.New(logTo, "", 0),
	})
}

// ---- RED-PROOF 3 — the both-due night ----------------------------------------------------

// EXACTLY ONE stop/start pair, with BOTH backups inside it. Asserting only "both backups ran"
// would pass against an implementation that quiesces twice — the count is the assertion.
//
// COMPANION RED-PROOF (observed): make runOnce call quiesceAndPollTiers once per due tier
// (a per-tier cycle instead of one window) → stops/starts become 2/2 and this fails with
// "want EXACTLY 1 stop and 1 start ... got stops=2 starts=2". Restored.
func TestBothTiersDue_ExactlyOneQuiesceWindow(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.phases["local"] = []string{phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("runOnce: %v", err)
	}
	stops, starts := len(st.stoppedNames()), len(st.startedNames())
	if stops != 1 || starts != 1 {
		t.Fatalf("both-due night must be ONE quiesce window: want EXACTLY 1 stop and 1 start, got stops=%d starts=%d (stopped=%v started=%v)",
			stops, starts, st.stoppedNames(), st.startedNames())
	}
	got := be.startedTargets()
	if len(got) != 2 || got[0] != "local" || got[1] != "felhom-pbs" {
		t.Fatalf("both tiers must back up, primary first: got %v", got)
	}
}

// One tier due → one quiesce, that tier only.
func TestOnlyPBSDue_OneQuiesceThatTierOnly(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["felhom-pbs"] = true
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	stops, starts := len(st.stoppedNames()), len(st.startedNames())
	if stops != 1 || starts != 1 {
		t.Fatalf("want 1/1, got stops=%d starts=%d", stops, starts)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("only the due tier may back up; got %v", got)
	}
}

// Neither due → no quiesce at all. The app must not be touched.
func TestNoTierDue_NoQuiesce(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 0 || starts != 0 {
		t.Fatalf("no tier due must not touch the app; got stops=%d starts=%d", stops, starts)
	}
	if got := be.startedTargets(); len(got) != 0 {
		t.Fatalf("no backup may start; got %v", got)
	}
}

// The app must stay DOWN until the LAST tier snapshots. Resuming after tier 1 would leave the DR
// tier capturing a running app — losing app-consistency on exactly the tier we most want it on.
func TestNonLastTierSnapshot_DoesNotResumeApp(t *testing.T) {
	st := &fakeStacks{running: []string{"immich"}}
	be := newTierBackend()
	be.stacks = st
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	// The local tier snapshots first, then finishes. The app must NOT come back at its snapshot.
	be.phases["local"] = []string{phaseSnapshotted, phaseSnapshotted, phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}

	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	// THE assertion: when the SECOND (last) tier started, zero restarts had happened — i.e. the app
	// was still quiesced. Resuming at tier 1's snapshot would leave the DR tier capturing a RUNNING
	// app, losing app-consistency on exactly the tier we most want it on.
	at := be.restartsWhenEachTierStarted()
	if len(at) != 2 {
		t.Fatalf("both tiers must start; sampled %v", at)
	}
	if at[1] != 0 {
		t.Fatalf("the app had ALREADY resumed (%d restarts) when the last tier started — non-last snapshot must not resume", at[1])
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("want exactly one stop/start pair; got %d/%d", stops, starts)
	}
}

// ---- RED-PROOF 2 — new controller ↔ OLD agent --------------------------------------------

// The agent 404s /backup/tiers. The controller MUST degrade to the untargeted tier, LOG it, and
// STILL TAKE A BACKUP.
//
// The hollow version of this test asserts only "runOnce returned nil" — which passes against an
// implementation that silently skips the backup entirely. The assertion that matters is that a
// backup actually started.
//
// COMPANION RED-PROOF (observed): make resolveDueTiers return (nil, false, nil) on
// ErrTiersUnsupported — i.e. treat "no tier support" as "nothing due" → this test fails with
// "OLD AGENT: a backup MUST still be taken; got started=[]". Restored.
func TestOldAgent_DegradesToUntargetedAndStillBacksUp(t *testing.T) {
	be := newTierBackend()
	be.tiersErr = ErrTiersUnsupported
	be.untargetedDue = true

	st := &fakeStacks{running: []string{"immich"}}
	var logbuf strings.Builder
	l := newTierLoop(t, be, st, &logbuf)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("runOnce against an old agent must not error: %v", err)
	}
	got := be.startedTargets()
	if len(got) != 1 || got[0] != "(untargeted)" {
		t.Fatalf("OLD AGENT: a backup MUST still be taken via the untargeted path; got started=%v", got)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("old-agent path must still be one quiesce window; got %d/%d", stops, starts)
	}
	if !strings.Contains(logbuf.String(), "predates R-82") {
		t.Fatalf("the degrade must be LOGGED — a silent degrade is indistinguishable from multi-tier working; log:\n%s", logbuf.String())
	}
}

// The degrade line is logged ONCE, not every poll (it is a steady state during a rollout).
func TestOldAgent_DegradeLoggedOnce(t *testing.T) {
	be := newTierBackend()
	be.tiersErr = ErrTiersUnsupported
	be.untargetedDue = false // not due → cheap repeated polls

	var logbuf strings.Builder
	l := newTierLoop(t, be, &fakeStacks{}, &logbuf)
	for i := 0; i < 5; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
	}
	if n := strings.Count(logbuf.String(), "predates R-82"); n != 1 {
		t.Fatalf("degrade must log exactly once across polls, got %d:\n%s", n, logbuf.String())
	}
}

// A backend that does not implement TieredBackend at all (an older controller build path) uses the
// untargeted route with no probe and no degrade log.
func TestPlainBackend_UsesUntargetedPath(t *testing.T) {
	be := &plainBackend{due: true}
	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if be.started != 1 {
		t.Fatalf("a plain Backend must still back up; started=%d", be.started)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("want 1/1, got %d/%d", stops, starts)
	}
}

type plainBackend struct {
	due     bool
	started int
}

func (p *plainBackend) Due(context.Context) (bool, *int64, error) { return p.due, nil, nil }
func (p *plainBackend) StartBackup(context.Context) (string, error) {
	p.started++
	return "job", nil
}
func (p *plainBackend) BackupStatus(context.Context) (string, error) { return phaseDone, nil }

// ---- resilience --------------------------------------------------------------------------

// One tier failing to START must not prevent the other tier's backup, and the app must still resume
// exactly once.
func TestOneTierFailsToStart_OtherStillRunsAndAppResumes(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.startErrOn = "local"
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	err := l.runOnce(context.Background())
	if err == nil {
		t.Fatal("a tier start failure must be reported, not swallowed")
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("the surviving tier must still back up; got %v", got)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("app must resume exactly once even when a tier fails; got %d/%d", stops, starts)
	}
}

// A tier whose due-check errors must not drop the OTHER tier's backup.
func TestDueCheckErrorOnOneTier_OtherTierStillEvaluated(t *testing.T) {
	be := &dueErrBackend{tierBackend: newTierBackend(), errOn: "local"}
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["felhom-pbs"] = true
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("a single tier's due-check failure must not fail the cycle: %v", err)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("the healthy tier must still back up; got %v", got)
	}
}

type dueErrBackend struct {
	*tierBackend
	errOn string
}

func (d *dueErrBackend) DueFor(ctx context.Context, target string) (bool, *int64, error) {
	if target == d.errOn {
		return false, nil, fmt.Errorf("simulated due-check failure")
	}
	return d.tierBackend.DueFor(ctx, target)
}

// An agent advertising ZERO tiers must fall back to the untargeted path, not do nothing.
func TestZeroTiersAdvertised_FallsBackNotSilent(t *testing.T) {
	be := newTierBackend()
	be.tiers = nil // advertised, but empty
	be.untargetedDue = true

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "(untargeted)" {
		t.Fatalf("zero advertised tiers must fall back, never skip; got %v", got)
	}
}

// oldestAge drives the window gate's safety valve: a NEVER-backed-up tier (nil age) must win over
// a fresher sibling, or a stale DR tier could be starved by a healthy local one.
func TestOldestAge(t *testing.T) {
	i := func(v int64) *int64 { return &v }
	if got := oldestAge([]dueTier{{ageSecs: i(10)}, {ageSecs: i(99)}}); got == nil || *got != 99 {
		t.Fatalf("want 99, got %v", got)
	}
	if got := oldestAge([]dueTier{{ageSecs: i(10)}, {ageSecs: nil}}); got != nil {
		t.Fatalf("a never-backed-up tier (nil) must win; got %v", *got)
	}
	if got := oldestAge(nil); got != nil {
		t.Fatalf("empty → nil, got %v", *got)
	}
}