Files
felhom-controller/controller/internal/quiesce/invariant_test.go
T
admin 86ea482fc1 controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a
licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY
agent, and refuses it for AgeStateUnknown. An unreadable storage no longer
masquerades as a first-ever backup and no longer quiesces apps outside the window.

A missing wire field means legacy, not unknown — deliberately. Treating it as
unknown would stop the valve firing on un-upgraded boxes and starve genuinely new
ones. Degrade logged once; unrecognised future values also map to legacy.

Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the
signature change compiled and vetted clean while quiesceBackend silently stopped
satisfying it — which would have degraded every box to the single-tier path with
no error. Added a compile-time witness.

Also corrects the notifier comment that claimed operator-only came from a missing
customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
2026-07-27 18:08:56 +02:00

148 lines
6.3 KiB
Go

package quiesce
import (
"context"
"testing"
"time"
)
// R-88 Part 3 — the invariant, pinned by name so a refactor has to delete an obviously-named
// contract to reintroduce the bug. See the comment block on scheduledRunAllowed.
//
// A missing value means UNKNOWN. Only a POSITIVE determination of "never backed up" may fire the
// safety valve. Prior instances of the opposite mistake: hub v0.12.0, v0.73.0, R-81, R-88.
//
// ── WHY ONLY HALF THE CONTRACT IS PINNED HERE ────────────────────────────────────────────────
//
// Scenario C ("unknown must NOT bypass the window gate") is NOT testable in this package today, and
// writing a test that pretends otherwise would be worse than leaving it out. The agent returns
// byte-identical responses for "the storage read errored" and "there has never been a backup" —
// same Due, same Reason, same nil AgeSecs — so the controller has nothing to discriminate on. Giving
// "unknown" its own representation is an agent wire change, tracked as R-88 Part 2.
//
// What IS pinned here is Scenario D, and it is the half that a careless Part 2 would break. A fix
// that makes nil stop firing the valve — the obvious way to "fix C" — turns a loud bug into a silent
// one: a box powered on only outside its backup window would never back up at all, and nobody would
// notice for weeks. This test fails against that implementation.
// agedBackend wraps tierBackend so DueFor can report a REAL age. The shared fake always returns nil
// ("never"), which is exactly the case under test here — so the control case needs its own shape
// rather than a field added to a fake three other suites depend on.
type agedBackend struct {
*tierBackend
ages map[string]*int64
}
func (a *agedBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
due, _, _, err := a.tierBackend.DueFor(ctx, target)
// A real age implies a KNOWN state — that is what the agent would send.
return due, a.ages[target], string(AgeStateKnown), err
}
// SCENARIO D — a genuinely never-backed-up box still gets its first backup, outside the window.
//
// COMPANION RED-PROOF (observed): flip the nil branch of scheduledRunAllowed from `return true` to
// `return false` — i.e. apply "unknown must not bypass the gate" too broadly, which is precisely the
// over-correction Scenario C invites — and this fails with:
//
// "CONTRACT VIOLATED: a never-backed-up box outside its window did NOT back up (0 stack stop(s)) —
// the safety valve was removed; a box only ever powered on outside its window would starve"
//
// (DELETING the branch outright instead panics on the nil deref two lines down — the test still
// catches it, but `return false` is the mutation a real over-correction would produce.)
//
// Restored.
func TestContract_NeverBackedUp_RunsOutsideTheWindow(t *testing.T) {
// 12:00 Budapest, window 02:30 → gate [04:30, 08:30). Firmly outside.
outside := atBudapest(12, 0)
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local"}}
be.dueSet["local"] = true
be.phases["local"] = []string{phaseDone}
// DueFor returns a nil age — "never backed up" (see tierBackend.DueFor).
st := &fakeStacks{running: []string{"bookstack"}}
l := windowLoop(t, be, st, "02:30", outside)
if err := l.runOnce(context.Background()); err != nil {
t.Fatalf("runOnce: %v", err)
}
if got := len(st.stoppedNames()); got == 0 {
t.Fatalf("CONTRACT VIOLATED: a never-backed-up box outside its window did NOT back up (%d stack stop(s)) — "+
"the safety valve was removed; a box only ever powered on outside its window would starve", got)
}
}
// The control for the test above: with a RECENT backup, the same cycle at the same hour defers.
// Without this, TestContract_NeverBackedUp_RunsOutsideTheWindow would also pass against a gate that
// was simply disabled.
func TestContract_RecentBackup_DefersOutsideTheWindow(t *testing.T) {
outside := atBudapest(12, 0)
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local"}}
be.dueSet["local"] = true
be.phases["local"] = []string{phaseDone}
aged := &agedBackend{tierBackend: be, ages: map[string]*int64{"local": i64(20 * 3600)}} // 20h old
st := &fakeStacks{running: []string{"bookstack"}}
l := windowLoop(t, aged, st, "02:30", outside)
if err := l.runOnce(context.Background()); err != nil {
t.Fatalf("runOnce: %v", err)
}
if got := len(st.stoppedNames()); got != 0 {
t.Fatalf("a recently-backed-up box outside its window must DEFER, got %d stack stop(s) — "+
"if this passes the gate is not actually gating anything", got)
}
}
// The valve boundary as a truth table on the real predicate, named so it cannot be quietly dropped.
func TestContract_SafetyValveBoundary(t *testing.T) {
const window = "02:30"
outside := atBudapest(12, 0)
h := func(hours int64) *int64 { return i64(hours * 3600) }
cases := []struct {
name string
age *int64
want bool
}{
{"never backed up (nil) → the valve FIRES", nil, true},
{"just inside cadence+24h → defer", h(47), false},
{"exactly at cadence+24h → defer (strictly greater)", h(48), false},
{"past cadence+24h → the valve fires", h(49), true},
}
for _, c := range cases {
if got := scheduledRunAllowed(outside, window, c.age, true, cadence24); got != c.want {
t.Errorf("CONTRACT VIOLATED: %s → scheduledRunAllowed = %v, want %v", c.name, got, c.want)
}
}
}
// The breaker must not be able to starve a first-ever backup either: a never-backed-up tier that
// fails still backs off (it must — that is the whole point), but the backoff EXPIRES and it retries.
func TestContract_BreakerNeverStarvesAFirstBackup(t *testing.T) {
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
be := tierBackendFailing("local", "local")
st := &fakeStacks{running: []string{"bookstack"}}
l := breakerLoop(t, be, st, &now)
attempts := 0
for i := 0; i < 4; i++ {
before := len(st.stoppedNames())
if err := l.runOnce(context.Background()); err != nil {
t.Fatal(err)
}
if len(st.stoppedNames()) > before {
attempts++
}
now = now.Add(backoffFor(i+1) + time.Minute) // step past each successive backoff
}
if attempts < 4 {
t.Fatalf("CONTRACT VIOLATED: a never-backed-up tier attempted only %d time(s) across 4 expired backoffs; "+
"the breaker must bound the RETRY INTERVAL, never stop retrying", attempts)
}
}