felhom-controller/controller/internal/quiesce/tiers_test.go

package quiesce

import (
	"context"
	"fmt"
	"io"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// R-82 Slice B — two tiers; since `09` §3 decision 156 (R-518 option A), one stop per tier.
//
// Two properties are load-bearing and neither is provable by "no error was returned":
//   1. Each window backs up ONE tier and resumes the apps at its snapshot; another due tier gets its
//      own window in a later cycle (decision 156 reverses R-82's one window for both tiers).
//   2. A new controller against an OLD agent still TAKES A BACKUP. The hollow version of that test
//      asserts "no error" while silently skipping the backup — the exact failure it exists to catch.

// ---- fakes -------------------------------------------------------------------------------

// NOTE: fakeStacks comes from quiesce_test.go (same package) — reused rather than duplicated.
// Counts come from len(stoppedNames()) / len(startedNames()).

// tierBackend is a multi-tier fake agent. phases[target] is the phase sequence returned by
// successive BackupStatusFor calls for that tier.
type tierBackend struct {
	mu            sync.Mutex
	tiers         []BackupTier
	tiersErr      error
	dueSet        map[string]bool
	phases        map[string][]string
	phaseIdx      map[string]int
	started       []string // targets StartBackupFor/StartBackup was called with, in order
	untargetedDue bool
	startErrOn    string
	// stacks (optional) lets a start sample how many restarts have happened SO FAR — the direct
	// way to assert "the app had not resumed when this tier started".
	stacks        *fakeStacks
	startsAtStart []int
	// stopsAtStart samples the stop count when each tier starts — a new window shows as a higher
	// stop count than the previous tier saw (R-518, decision 156).
	stopsAtStart []int
	// statusRestarts[target] samples the restart count at EACH status poll of that tier, so a test can
	// say "the apps were already started when the Nth poll answered" (R-518, decision 156).
	statusRestarts map[string][]int
}

func newTierBackend() *tierBackend {
	return &tierBackend{
		dueSet: map[string]bool{}, phases: map[string][]string{}, phaseIdx: map[string]int{},
		statusRestarts: map[string][]int{},
	}
}

func (b *tierBackend) Tiers(context.Context) ([]BackupTier, error) {
	if b.tiersErr != nil {
		return nil, b.tiersErr
	}
	return b.tiers, nil
}

// DueFor returns a nil age with an EMPTY age_state — i.e. the pre-v0.105.0 (legacy) shape, which
// keeps every suite written before R-88 Part 2 asserting exactly the behaviour it always did.
func (b *tierBackend) DueFor(_ context.Context, target string) (bool, *int64, string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.dueSet[target], nil, "", nil
}
func (b *tierBackend) StartBackupFor(_ context.Context, target string) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.startErrOn == target {
		return "", fmt.Errorf("simulated start failure on %s", target)
	}
	b.started = append(b.started, target)
	if b.stacks != nil {
		b.startsAtStart = append(b.startsAtStart, len(b.stacks.startedNames()))
		b.stopsAtStart = append(b.stopsAtStart, len(b.stacks.stoppedNames()))
	}
	return "job-" + target, nil
}
func (b *tierBackend) BackupStatusFor(_ context.Context, target string) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.stacks != nil {
		b.statusRestarts[target] = append(b.statusRestarts[target], len(b.stacks.startedNames()))
	}
	seq := b.phases[target]
	i := b.phaseIdx[target]
	if i < len(seq) {
		b.phaseIdx[target]++
		return seq[i], nil
	}
	if len(seq) > 0 {
		return seq[len(seq)-1], nil
	}
	return phaseDone, nil
}

// The untargeted (pre-R-82) surface.
func (b *tierBackend) Due(context.Context) (bool, *int64, error) {
	return b.untargetedDue, nil, nil
}
func (b *tierBackend) StartBackup(context.Context) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.started = append(b.started, "(untargeted)")
	return "job-untargeted", nil
}
func (b *tierBackend) BackupStatus(context.Context) (string, error) { return phaseDone, nil }

func (b *tierBackend) restartsWhenEachTierStarted() []int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]int(nil), b.startsAtStart...)
}

func (b *tierBackend) stopsWhenEachTierStarted() []int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]int(nil), b.stopsAtStart...)
}

func (b *tierBackend) restartsAtEachPoll(target string) []int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]int(nil), b.statusRestarts[target]...)
}

func (b *tierBackend) setDue(target string, due bool) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.dueSet[target] = due
}

func (b *tierBackend) startedTargets() []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]string(nil), b.started...)
}

func newTierLoop(t *testing.T, be Backend, st *fakeStacks, logTo io.Writer) *Loop {
	t.Helper()
	if logTo == nil {
		logTo = io.Discard
	}
	return New(Options{
		Backend:    be,
		Stacks:     st,
		MarkerPath: filepath.Join(t.TempDir(), "quiesce-state.json"),
		StatusPoll: time.Millisecond,
		MaxQuiesce: 30 * time.Second,
		Logger:     log.New(logTo, "", 0),
	})
}

// ---- R-518 option A — one stop per backup tier (`09` §3 decision 156) --------------------
//
// Decision 156 REVERSES R-82's „ONE quiesce window for both due tiers (never two app outages for one
// night)". The one long window kept every app down for the whole local upload (5 min 47 s measured on
// demo-hp, 9 apps). Now each window runs ONLY the first due tier and resumes the apps at its
// `snapshotted`; another due tier waits for a LATER cycle and gets its own short window. Every copy is
// still taken with the apps stopped, so each stays app-consistent.

// Both tiers due → the first cycle stops once, backs up ONLY the first (primary) tier, starts once.
// The other tier is not started in this cycle — never straight after tier 1 (the next poll picks it
// up, pinned by TestLeftoverTier_RunsInNextCycleInItsOwnWindow).
//
// Was TestBothTiersDue_ExactlyOneQuiesceWindow (R-82), which pinned both tiers inside one window.
func TestBothTiersDue_FirstCycleRunsOnlyFirstTier(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.phases["local"] = []string{phaseSnapshotted, phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("runOnce: %v", err)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("one window: want EXACTLY 1 stop and 1 start, got stops=%d starts=%d (stopped=%v started=%v)",
			stops, starts, st.stoppedNames(), st.startedNames())
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
		t.Fatalf("decision 156: a window backs up ONLY the first due tier (primary); got %v", got)
	}
}

// THE red test of R-518: the apps must run while the local copy still uploads. Local reports
// `snapshotted` three times, then `done`; the stacks must already have been STARTED when the SECOND
// `snapshotted` poll answered. Under R-82 the app stayed down until the LAST tier snapshotted, so the
// whole local upload ran with every app stopped.
//
// Was TestNonLastTierSnapshot_DoesNotResumeApp (R-82), which pinned the opposite (decision 156
// reverses R-82's one window for both tiers).
func TestFirstTierSnapshot_ResumesAppWhileUploadContinues(t *testing.T) {
	st := &fakeStacks{running: []string{"immich"}}
	be := newTierBackend()
	be.stacks = st
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.phases["local"] = []string{phaseSnapshotted, phaseSnapshotted, phaseSnapshotted, phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}

	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	polls := be.restartsAtEachPoll("local")
	if len(polls) < 2 {
		t.Fatalf("the local tier must be polled past its first snapshot; restarts per poll = %v", polls)
	}
	if polls[1] < 1 {
		t.Fatalf("the apps were still STOPPED when the second `snapshotted` poll answered (restarts per poll = %v) — they must resume at the first tier's snapshot (decision 156)", polls)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("want exactly one stop/start pair; got %d/%d", stops, starts)
	}
}

// The tier left over by a cycle IS run by the NEXT cycle, in its OWN window (a second stop that comes
// after the first window's start). Without this, decision 156 would silently drop the off-site copy.
func TestLeftoverTier_RunsInNextCycleInItsOwnWindow(t *testing.T) {
	st := &fakeStacks{running: []string{"immich"}}
	be := newTierBackend()
	be.stacks = st
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.phases["local"] = []string{phaseSnapshotted, phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}
	l := newTierLoop(t, be, st, nil)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("cycle 1: %v", err)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
		t.Fatalf("cycle 1 must run only the local tier; got %v", got)
	}
	// The agent now reports local as backed up; the off-site tier is still due.
	be.setDue("local", false)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("cycle 2: %v", err)
	}
	got := be.startedTargets()
	if len(got) != 2 || got[1] != "felhom-pbs" {
		t.Fatalf("cycle 2 must run the left-over off-site tier; got %v", got)
	}
	// Its own window: when it started, the first window had already resumed (1 start) and a NEW stop
	// had happened (2 stops).
	stopsAt, startsAt := be.stopsWhenEachTierStarted(), be.restartsWhenEachTierStarted()
	if stopsAt[1] != 2 || startsAt[1] != 1 {
		t.Fatalf("the off-site tier must start in its OWN window; at its start stops=%d starts=%d (want 2/1)", stopsAt[1], startsAt[1])
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 2 || starts != 2 {
		t.Fatalf("two cycles, two windows: want 2 stops / 2 starts, got %d/%d", stops, starts)
	}
	// A third cycle with nothing due touches nothing.
	be.setDue("felhom-pbs", false)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if stops := len(st.stoppedNames()); stops != 2 {
		t.Fatalf("nothing due must not stop the apps again; stops=%d", stops)
	}
}

// A manual press („Mentés most") makes the LOCAL copy only, in one short stop; the off-site tier is
// NOT requested — it follows at the next scheduled night run (decision 156).
func TestManualPress_RunsOnlyLocalTier(t *testing.T) {
	st := &fakeStacks{running: []string{"immich"}}
	be := newTierBackend()
	be.stacks = st
	// Off-site listed FIRST on purpose: the press must pick the tier the agent flags Primary, not
	// whichever comes first.
	be.tiers = []BackupTier{{Target: "felhom-pbs"}, {Target: "local", Primary: true}}
	be.phases["local"] = []string{phaseSnapshotted, phaseDone}
	be.phases["felhom-pbs"] = []string{phaseSnapshotted, phaseDone}
	l := newTierLoop(t, be, st, nil)

	if err := l.TriggerNow(); err != nil {
		t.Fatalf("TriggerNow: %v", err)
	}
	// TriggerNow holds l.mu for the whole async cycle; taking it waits for the cycle to finish.
	l.mu.Lock()
	l.mu.Unlock() //nolint:staticcheck // deliberate: wait for the async cycle

	if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
		t.Fatalf("a manual press must back up ONLY the local (primary) tier; started=%v", got)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("a manual press is one short stop; got %d/%d", stops, starts)
	}
}

// One tier due → one quiesce, that tier only.
func TestOnlyPBSDue_OneQuiesceThatTierOnly(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["felhom-pbs"] = true
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	stops, starts := len(st.stoppedNames()), len(st.startedNames())
	if stops != 1 || starts != 1 {
		t.Fatalf("want 1/1, got stops=%d starts=%d", stops, starts)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("only the due tier may back up; got %v", got)
	}
}

// Neither due → no quiesce at all. The app must not be touched.
func TestNoTierDue_NoQuiesce(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 0 || starts != 0 {
		t.Fatalf("no tier due must not touch the app; got stops=%d starts=%d", stops, starts)
	}
	if got := be.startedTargets(); len(got) != 0 {
		t.Fatalf("no backup may start; got %v", got)
	}
}

// ---- RED-PROOF 2 — new controller ↔ OLD agent --------------------------------------------

// The agent 404s /backup/tiers. The controller MUST degrade to the untargeted tier, LOG it, and
// STILL TAKE A BACKUP.
//
// The hollow version of this test asserts only "runOnce returned nil" — which passes against an
// implementation that silently skips the backup entirely. The assertion that matters is that a
// backup actually started.
//
// COMPANION RED-PROOF (observed): make resolveDueTiers return (nil, false, nil) on
// ErrTiersUnsupported — i.e. treat "no tier support" as "nothing due" → this test fails with
// "OLD AGENT: a backup MUST still be taken; got started=[]". Restored.
func TestOldAgent_DegradesToUntargetedAndStillBacksUp(t *testing.T) {
	be := newTierBackend()
	be.tiersErr = ErrTiersUnsupported
	be.untargetedDue = true

	st := &fakeStacks{running: []string{"immich"}}
	var logbuf strings.Builder
	l := newTierLoop(t, be, st, &logbuf)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("runOnce against an old agent must not error: %v", err)
	}
	got := be.startedTargets()
	if len(got) != 1 || got[0] != "(untargeted)" {
		t.Fatalf("OLD AGENT: a backup MUST still be taken via the untargeted path; got started=%v", got)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("old-agent path must still be one quiesce window; got %d/%d", stops, starts)
	}
	if !strings.Contains(logbuf.String(), "predates R-82") {
		t.Fatalf("the degrade must be LOGGED — a silent degrade is indistinguishable from multi-tier working; log:\n%s", logbuf.String())
	}
}

// The degrade line is logged ONCE, not every poll (it is a steady state during a rollout).
func TestOldAgent_DegradeLoggedOnce(t *testing.T) {
	be := newTierBackend()
	be.tiersErr = ErrTiersUnsupported
	be.untargetedDue = false // not due → cheap repeated polls

	var logbuf strings.Builder
	l := newTierLoop(t, be, &fakeStacks{}, &logbuf)
	for i := 0; i < 5; i++ {
		if err := l.runOnce(context.Background()); err != nil {
			t.Fatal(err)
		}
	}
	if n := strings.Count(logbuf.String(), "predates R-82"); n != 1 {
		t.Fatalf("degrade must log exactly once across polls, got %d:\n%s", n, logbuf.String())
	}
}

// A backend that does not implement TieredBackend at all (an older controller build path) uses the
// untargeted route with no probe and no degrade log.
func TestPlainBackend_UsesUntargetedPath(t *testing.T) {
	be := &plainBackend{due: true}
	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if be.started != 1 {
		t.Fatalf("a plain Backend must still back up; started=%d", be.started)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("want 1/1, got %d/%d", stops, starts)
	}
}

type plainBackend struct {
	due     bool
	started int
}

func (p *plainBackend) Due(context.Context) (bool, *int64, error) { return p.due, nil, nil }
func (p *plainBackend) StartBackup(context.Context) (string, error) {
	p.started++
	return "job", nil
}
func (p *plainBackend) BackupStatus(context.Context) (string, error) { return phaseDone, nil }

// ---- resilience --------------------------------------------------------------------------

// One tier failing to START must not prevent the other tier's backup, and the app must still resume
// exactly once.
func TestOneTierFailsToStart_OtherStillRunsAndAppResumes(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	be.startErrOn = "local"
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	err := l.runOnce(context.Background())
	if err == nil {
		t.Fatal("a tier start failure must be reported, not swallowed")
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("the surviving tier must still back up; got %v", got)
	}
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("app must resume exactly once even when a tier fails; got %d/%d", stops, starts)
	}
}

// A tier whose due-check errors must not drop the OTHER tier's backup.
func TestDueCheckErrorOnOneTier_OtherTierStillEvaluated(t *testing.T) {
	be := &dueErrBackend{tierBackend: newTierBackend(), errOn: "local"}
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["felhom-pbs"] = true
	be.phases["felhom-pbs"] = []string{phaseDone}

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("a single tier's due-check failure must not fail the cycle: %v", err)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "felhom-pbs" {
		t.Fatalf("the healthy tier must still back up; got %v", got)
	}
}

type dueErrBackend struct {
	*tierBackend
	errOn string
}

func (d *dueErrBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
	if target == d.errOn {
		return false, nil, "", fmt.Errorf("simulated due-check failure")
	}
	return d.tierBackend.DueFor(ctx, target)
}

// An agent advertising ZERO tiers must fall back to the untargeted path, not do nothing.
func TestZeroTiersAdvertised_FallsBackNotSilent(t *testing.T) {
	be := newTierBackend()
	be.tiers = nil // advertised, but empty
	be.untargetedDue = true

	st := &fakeStacks{running: []string{"immich"}}
	l := newTierLoop(t, be, st, nil)
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatal(err)
	}
	if got := be.startedTargets(); len(got) != 1 || got[0] != "(untargeted)" {
		t.Fatalf("zero advertised tiers must fall back, never skip; got %v", got)
	}
}

// oldestAge drives the window gate's safety valve: a NEVER-backed-up tier (nil age) must win over
// a fresher sibling, or a stale DR tier could be starved by a healthy local one.
func TestOldestAge(t *testing.T) {
	i := func(v int64) *int64 { return &v }
	if got := oldestAge([]dueTier{{ageSecs: i(10)}, {ageSecs: i(99)}}); got == nil || *got != 99 {
		t.Fatalf("want 99, got %v", got)
	}
	if got := oldestAge([]dueTier{{ageSecs: i(10)}, {ageSecs: nil}}); got != nil {
		t.Fatalf("a never-backed-up tier (nil) must win; got %v", *got)
	}
	if got := oldestAge(nil); got != nil {
		t.Fatalf("empty → nil, got %v", *got)
	}
}

// Operator ruling 2026-07-26: "let the first backup run as long as needed; other backups shouldn't
// start until finished." A first FULL offsite snapshot legitimately runs for hours — far past the
// quiesce bound. When that bound elapses the app resumes (correct), but the remaining tiers must
// NOT start: vzdump still holds the guest lock, so a second start would be refused by the agent
// (409) or fail on the lock and record a spurious failure that leaves the tier permanently due.
//
// COMPANION RED-PROOF (observed): drop the `break` on stillRunning → the second tier starts while
// the first is still going and this fails with
// "the second tier MUST NOT start while the first is still running; started=[local felhom-pbs]".
// Restored.
func TestTierOverrunsQuiesceBound_RemainingTiersDeferred(t *testing.T) {
	be := newTierBackend()
	be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
	be.dueSet["local"] = true
	be.dueSet["felhom-pbs"] = true
	// The local tier never reaches a terminal phase — it just keeps running, like a long upload.
	be.phases["local"] = []string{phaseRunningForever}

	st := &fakeStacks{running: []string{"immich"}}
	l := New(Options{
		Backend: be, Stacks: st,
		MarkerPath: filepath.Join(t.TempDir(), "q.json"),
		StatusPoll: time.Millisecond,
		MaxQuiesce: 20 * time.Millisecond, // bound elapses almost immediately
		Logger:     log.New(io.Discard, "", 0),
	})
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("an overrun is not an error: %v", err)
	}
	got := be.startedTargets()
	if len(got) != 1 || got[0] != "local" {
		t.Fatalf("the second tier MUST NOT start while the first is still running; started=%v", got)
	}
	// The app must still have come back exactly once (the max-quiesce guard).
	if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
		t.Fatalf("the app must resume exactly once on the quiesce bound; got %d/%d", stops, starts)
	}
}

// phaseRunningForever is a non-terminal phase the fake returns indefinitely.
const phaseRunningForever = "running"