v0.175.0 — R-82: a tier that overruns the quiesce bound defers the rest
Operator ruling 2026-07-26: let the first backup run as long as needed; other backups shouldn't start until finished. A first FULL offsite snapshot runs for hours, far past max_quiesce. When that bound elapses the app resumes (unchanged), but the loop then started the NEXT tier while the first was still uploading. Now it breaks and defers the rest to a later poll — vzdump still holds the guest lock, so the second start would be refused by the agent (409, v0.99.0) or fail on the lock, and a failed backup never satisfies a cadence, so the tier would retry into the same wall forever. pollTier returns (phase, stillRunning, err). The app still resumes exactly once. Red-proof observed and restored; full suite green.
This commit is contained in:
@@ -397,3 +397,45 @@ func TestOldestAge(t *testing.T) {
|
||||
t.Fatalf("empty → nil, got %v", *got)
|
||||
}
|
||||
}
|
||||
|
||||
// Operator ruling 2026-07-26: "let the first backup run as long as needed; other backups shouldn't
|
||||
// start until finished." A first FULL offsite snapshot legitimately runs for hours — far past the
|
||||
// quiesce bound. When that bound elapses the app resumes (correct), but the remaining tiers must
|
||||
// NOT start: vzdump still holds the guest lock, so a second start would be refused by the agent
|
||||
// (409) or fail on the lock and record a spurious failure that leaves the tier permanently due.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop the `break` on stillRunning → the second tier starts while
|
||||
// the first is still going and this fails with
|
||||
// "the second tier MUST NOT start while the first is still running; started=[local felhom-pbs]".
|
||||
// Restored.
|
||||
func TestTierOverrunsQuiesceBound_RemainingTiersDeferred(t *testing.T) {
|
||||
be := newTierBackend()
|
||||
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
|
||||
be.dueSet["local"] = true
|
||||
be.dueSet["felhom-pbs"] = true
|
||||
// The local tier never reaches a terminal phase — it just keeps running, like a long upload.
|
||||
be.phases["local"] = []string{phaseRunningForever}
|
||||
|
||||
st := &fakeStacks{running: []string{"immich"}}
|
||||
l := New(Options{
|
||||
Backend: be, Stacks: st,
|
||||
MarkerPath: filepath.Join(t.TempDir(), "q.json"),
|
||||
StatusPoll: time.Millisecond,
|
||||
MaxQuiesce: 20 * time.Millisecond, // bound elapses almost immediately
|
||||
Logger: log.New(io.Discard, "", 0),
|
||||
})
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatalf("an overrun is not an error: %v", err)
|
||||
}
|
||||
got := be.startedTargets()
|
||||
if len(got) != 1 || got[0] != "local" {
|
||||
t.Fatalf("the second tier MUST NOT start while the first is still running; started=%v", got)
|
||||
}
|
||||
// The app must still have come back exactly once (the max-quiesce guard).
|
||||
if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
|
||||
t.Fatalf("the app must resume exactly once on the quiesce bound; got %d/%d", stops, starts)
|
||||
}
|
||||
}
|
||||
|
||||
// phaseRunningForever is a non-terminal phase the fake returns indefinitely.
|
||||
const phaseRunningForever = "running"
|
||||
|
||||
Reference in New Issue
Block a user