86ea482fc1
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY agent, and refuses it for AgeStateUnknown. An unreadable storage no longer masquerades as a first-ever backup and no longer quiesces apps outside the window. A missing wire field means legacy, not unknown — deliberately. Treating it as unknown would stop the valve firing on un-upgraded boxes and starve genuinely new ones. Degrade logged once; unrecognised future values also map to legacy. Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the signature change compiled and vetted clean while quiesceBackend silently stopped satisfying it — which would have degraded every box to the single-tier path with no error. Added a compile-time witness. Also corrects the notifier comment that claimed operator-only came from a missing customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
148 lines
6.3 KiB
Go
148 lines
6.3 KiB
Go
package quiesce
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// R-88 Part 3 — the invariant, pinned by name so a refactor has to delete an obviously-named
|
|
// contract to reintroduce the bug. See the comment block on scheduledRunAllowed.
|
|
//
|
|
// A missing value means UNKNOWN. Only a POSITIVE determination of "never backed up" may fire the
|
|
// safety valve. Prior instances of the opposite mistake: hub v0.12.0, v0.73.0, R-81, R-88.
|
|
//
|
|
// ── WHY ONLY HALF THE CONTRACT IS PINNED HERE ────────────────────────────────────────────────
|
|
//
|
|
// Scenario C ("unknown must NOT bypass the window gate") is NOT testable in this package today, and
|
|
// writing a test that pretends otherwise would be worse than leaving it out. The agent returns
|
|
// byte-identical responses for "the storage read errored" and "there has never been a backup" —
|
|
// same Due, same Reason, same nil AgeSecs — so the controller has nothing to discriminate on. Giving
|
|
// "unknown" its own representation is an agent wire change, tracked as R-88 Part 2.
|
|
//
|
|
// What IS pinned here is Scenario D, and it is the half that a careless Part 2 would break. A fix
|
|
// that makes nil stop firing the valve — the obvious way to "fix C" — turns a loud bug into a silent
|
|
// one: a box powered on only outside its backup window would never back up at all, and nobody would
|
|
// notice for weeks. This test fails against that implementation.
|
|
|
|
// agedBackend wraps tierBackend so DueFor can report a REAL age. The shared fake always returns nil
|
|
// ("never"), which is exactly the case under test here — so the control case needs its own shape
|
|
// rather than a field added to a fake three other suites depend on.
|
|
type agedBackend struct {
|
|
*tierBackend
|
|
ages map[string]*int64
|
|
}
|
|
|
|
func (a *agedBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
|
due, _, _, err := a.tierBackend.DueFor(ctx, target)
|
|
// A real age implies a KNOWN state — that is what the agent would send.
|
|
return due, a.ages[target], string(AgeStateKnown), err
|
|
}
|
|
|
|
// SCENARIO D — a genuinely never-backed-up box still gets its first backup, outside the window.
|
|
//
|
|
// COMPANION RED-PROOF (observed): flip the nil branch of scheduledRunAllowed from `return true` to
|
|
// `return false` — i.e. apply "unknown must not bypass the gate" too broadly, which is precisely the
|
|
// over-correction Scenario C invites — and this fails with:
|
|
//
|
|
// "CONTRACT VIOLATED: a never-backed-up box outside its window did NOT back up (0 stack stop(s)) —
|
|
// the safety valve was removed; a box only ever powered on outside its window would starve"
|
|
//
|
|
// (DELETING the branch outright instead panics on the nil deref two lines down — the test still
|
|
// catches it, but `return false` is the mutation a real over-correction would produce.)
|
|
//
|
|
// Restored.
|
|
func TestContract_NeverBackedUp_RunsOutsideTheWindow(t *testing.T) {
|
|
// 12:00 Budapest, window 02:30 → gate [04:30, 08:30). Firmly outside.
|
|
outside := atBudapest(12, 0)
|
|
|
|
be := newTierBackend()
|
|
be.tiers = []BackupTier{{Target: "local"}}
|
|
be.dueSet["local"] = true
|
|
be.phases["local"] = []string{phaseDone}
|
|
// DueFor returns a nil age — "never backed up" (see tierBackend.DueFor).
|
|
|
|
st := &fakeStacks{running: []string{"bookstack"}}
|
|
l := windowLoop(t, be, st, "02:30", outside)
|
|
|
|
if err := l.runOnce(context.Background()); err != nil {
|
|
t.Fatalf("runOnce: %v", err)
|
|
}
|
|
if got := len(st.stoppedNames()); got == 0 {
|
|
t.Fatalf("CONTRACT VIOLATED: a never-backed-up box outside its window did NOT back up (%d stack stop(s)) — "+
|
|
"the safety valve was removed; a box only ever powered on outside its window would starve", got)
|
|
}
|
|
}
|
|
|
|
// The control for the test above: with a RECENT backup, the same cycle at the same hour defers.
|
|
// Without this, TestContract_NeverBackedUp_RunsOutsideTheWindow would also pass against a gate that
|
|
// was simply disabled.
|
|
func TestContract_RecentBackup_DefersOutsideTheWindow(t *testing.T) {
|
|
outside := atBudapest(12, 0)
|
|
|
|
be := newTierBackend()
|
|
be.tiers = []BackupTier{{Target: "local"}}
|
|
be.dueSet["local"] = true
|
|
be.phases["local"] = []string{phaseDone}
|
|
aged := &agedBackend{tierBackend: be, ages: map[string]*int64{"local": i64(20 * 3600)}} // 20h old
|
|
|
|
st := &fakeStacks{running: []string{"bookstack"}}
|
|
l := windowLoop(t, aged, st, "02:30", outside)
|
|
|
|
if err := l.runOnce(context.Background()); err != nil {
|
|
t.Fatalf("runOnce: %v", err)
|
|
}
|
|
if got := len(st.stoppedNames()); got != 0 {
|
|
t.Fatalf("a recently-backed-up box outside its window must DEFER, got %d stack stop(s) — "+
|
|
"if this passes the gate is not actually gating anything", got)
|
|
}
|
|
}
|
|
|
|
// The valve boundary as a truth table on the real predicate, named so it cannot be quietly dropped.
|
|
func TestContract_SafetyValveBoundary(t *testing.T) {
|
|
const window = "02:30"
|
|
outside := atBudapest(12, 0)
|
|
h := func(hours int64) *int64 { return i64(hours * 3600) }
|
|
|
|
cases := []struct {
|
|
name string
|
|
age *int64
|
|
want bool
|
|
}{
|
|
{"never backed up (nil) → the valve FIRES", nil, true},
|
|
{"just inside cadence+24h → defer", h(47), false},
|
|
{"exactly at cadence+24h → defer (strictly greater)", h(48), false},
|
|
{"past cadence+24h → the valve fires", h(49), true},
|
|
}
|
|
for _, c := range cases {
|
|
if got := scheduledRunAllowed(outside, window, c.age, true, cadence24); got != c.want {
|
|
t.Errorf("CONTRACT VIOLATED: %s → scheduledRunAllowed = %v, want %v", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The breaker must not be able to starve a first-ever backup either: a never-backed-up tier that
|
|
// fails still backs off (it must — that is the whole point), but the backoff EXPIRES and it retries.
|
|
func TestContract_BreakerNeverStarvesAFirstBackup(t *testing.T) {
|
|
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
|
be := tierBackendFailing("local", "local")
|
|
st := &fakeStacks{running: []string{"bookstack"}}
|
|
l := breakerLoop(t, be, st, &now)
|
|
|
|
attempts := 0
|
|
for i := 0; i < 4; i++ {
|
|
before := len(st.stoppedNames())
|
|
if err := l.runOnce(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(st.stoppedNames()) > before {
|
|
attempts++
|
|
}
|
|
now = now.Add(backoffFor(i+1) + time.Minute) // step past each successive backoff
|
|
}
|
|
if attempts < 4 {
|
|
t.Fatalf("CONTRACT VIOLATED: a never-backed-up tier attempted only %d time(s) across 4 expired backoffs; "+
|
|
"the breaker must bound the RETRY INTERVAL, never stop retrying", attempts)
|
|
}
|
|
}
|