Files
felhom-controller/controller/internal/quiesce/tier_skip_test.go
T
admin d8c2634472 R-518 option A: one stop per backup tier (09 §3 decision 156)
Each quiesce window now backs up ONLY the first due tier that starts and
resumes the apps at its snapshotted; the upload finishes with the apps
running. Any other due tier stays due and a later cycle (the next poll)
takes it in its own short window - never straight after the first.
This reverses R-82's 'ONE quiesce window for both due tiers'.

A manual press (TriggerNow) backs up the primary (local) tier only, picked
by the agent's Primary flag; the off-site tier follows at the next
scheduled night run. If no available tier is flagged primary, the first
available tier is backed up rather than nothing.

Unchanged: marker written before any stop, one unquiesce per window, the
max-quiesce bound, BUSY/start-error handling (the next tier is still tried
in the same window when a tier does not start), breaker and contention.

Tests: TestBothTiersDue_ExactlyOneQuiesceWindow ->
TestBothTiersDue_FirstCycleRunsOnlyFirstTier; TestNonLastTierSnapshot_
DoesNotResumeApp -> TestFirstTierSnapshot_ResumesAppWhileUploadContinues
(red on old code); new TestManualPress_RunsOnlyLocalTier (red on old
code), TestLeftoverTier_RunsInNextCycleInItsOwnWindow,
TestManualRunTiers_NoPrimaryAvailable_BacksUpFirstAvailable;
TestNotify_BothFailingTiersAreReported now runs two cycles.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-06 15:15:33 +02:00

94 lines
3.7 KiB
Go

package quiesce
import (
"context"
"sync"
"testing"
"time"
)
// R-518 (cheap half) — a tier whose storage the agent reports ABSENT is not attempted, and no app is
// stopped for it. BIGNIGHT: the apps stayed stopped through a felhom-pbs tier whose storage did not
// exist, then that tier failed.
type skipRecorder struct {
mu sync.Mutex
skipped []string
}
func (r *skipRecorder) BackupFailed(string, string, string) {}
func (r *skipRecorder) BackupRecovered(string, string) {}
func (r *skipRecorder) BackupTierSkipped(tier, _ string) {
r.mu.Lock()
defer r.mu.Unlock()
r.skipped = append(r.skipped, tier)
}
// RED-PROOF (run 2026-09-15, recorded in REPORT.md): with skipAbsentTiers returning its input
// unchanged, the manual run started [local felhom-pbs] and this failed at "absent tier was attempted".
func TestManualRun_AbsentTierSkipped(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
be.phases["local"] = []string{phaseDone}
st := &fakeStacks{running: []string{"immich"}}
l := newTierLoop(t, be, st, nil)
rec := &skipRecorder{}
l.SetTierNotifier(rec)
if err := l.quiesceAndPollTiers(context.Background(), l.allTiersForManualRun(context.Background())); err != nil {
t.Fatalf("manual cycle: %v", err)
}
if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
t.Fatalf("absent tier was attempted: started=%v", got)
}
if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
t.Fatalf("want one stop/start for the local tier, got %d/%d", stops, starts)
}
// A second resolution in the same absence does not notify again.
_ = l.allTiersForManualRun(context.Background())
if len(rec.skipped) != 1 || rec.skipped[0] != "felhom-pbs" {
t.Fatalf("want exactly one skip notice for felhom-pbs, got %v", rec.skipped)
}
}
// Only the absent tier is due → no quiesce at all.
func TestScheduled_OnlyAbsentTierDue_NoQuiesce(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
be.dueSet["felhom-pbs"] = true
st := &fakeStacks{running: []string{"immich"}}
l := newTierLoop(t, be, st, nil)
l.now = func() time.Time { return time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC) }
_ = l.runOnce(context.Background())
if stops := len(st.stoppedNames()); stops != 0 {
t.Fatalf("an app was stopped for a tier whose storage does not exist (stops=%d)", stops)
}
if got := be.startedTargets(); len(got) != 0 {
t.Fatalf("absent tier started: %v", got)
}
}
// Unknown/legacy storage (StorageAbsent=false) is never skipped — fail toward backing up.
func TestManualRun_UnknownStorageNotSkipped(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
be.phases["local"] = []string{phaseDone}
be.phases["felhom-pbs"] = []string{phaseDone}
l := newTierLoop(t, be, &fakeStacks{running: []string{"immich"}}, nil)
if got := l.allTiersForManualRun(context.Background()); len(got) != 2 {
t.Fatalf("a tier without an absent verdict was dropped: %v", got)
}
}
// `09` §3 decision 156: a press backs up ONE tier. When no available tier is flagged primary (the
// primary's storage is absent), the first available tier is backed up — never nothing.
func TestManualRunTiers_NoPrimaryAvailable_BacksUpFirstAvailable(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true, StorageAbsent: true}, {Target: "felhom-pbs"}, {Target: "other"}}
l := newTierLoop(t, be, &fakeStacks{}, nil)
got := l.manualRunTiers(context.Background())
if len(got) != 1 || got[0].target != "felhom-pbs" {
t.Fatalf("want exactly the first available tier, got %+v", got)
}
}