controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY agent, and refuses it for AgeStateUnknown. An unreadable storage no longer masquerades as a first-ever backup and no longer quiesces apps outside the window. A missing wire field means legacy, not unknown — deliberately. Treating it as unknown would stop the valve firing on un-upgraded boxes and starve genuinely new ones. Degrade logged once; unrecognised future values also map to legacy. Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the signature change compiled and vetted clean while quiesceBackend silently stopped satisfying it — which would have degraded every box to the single-tier path with no error. Added a compile-time witness. Also corrects the notifier comment that claimed operator-only came from a missing customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
This commit is contained in:
@@ -1741,9 +1741,11 @@ func (b quiesceBackend) Tiers(ctx context.Context) ([]quiesce.BackupTier, error)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64, error) {
|
||||
func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
||||
r, err := b.c.BackupDueFor(ctx, target)
|
||||
return r.Due, r.AgeSecs, err
|
||||
// AgeState is passed through RAW; quiesce.ageStateFromWire owns the mapping, including the
|
||||
// legacy-vs-unknown distinction. An empty string here means a pre-v0.105.0 agent.
|
||||
return r.Due, r.AgeSecs, r.AgeState, err
|
||||
}
|
||||
func (b quiesceBackend) StartBackupFor(ctx context.Context, target string) (string, error) {
|
||||
r, err := b.c.StartBackupFor(ctx, target)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/quiesce"
|
||||
)
|
||||
|
||||
// R-88 Part 2 — TieredBackend is satisfied by a RUNTIME type assertion in `resolveDueTiers`
|
||||
// (`l.backend.(TieredBackend)`), NOT at compile time. So when DueFor's signature changed, the whole
|
||||
// repo still built and vetted clean while `quiesceBackend` silently stopped satisfying the
|
||||
// interface — which would have degraded every box to the untargeted single-tier path, losing R-82's
|
||||
// multi-tier backups entirely, with no error anywhere.
|
||||
//
|
||||
// This compile-time assertion is the only thing that catches it. It caught it during R-88 Part 2.
|
||||
// Do not delete it; a runtime-asserted interface needs a compile-time witness.
|
||||
func TestQuiesceBackendSatisfiesTieredBackend(t *testing.T) {
|
||||
var _ quiesce.TieredBackend = quiesceBackend{}
|
||||
}
|
||||
@@ -174,6 +174,14 @@ type DueResponse struct {
|
||||
Due bool `json:"due"`
|
||||
Reason string `json:"reason"`
|
||||
AgeSecs *int64 `json:"age_seconds"`
|
||||
// AgeState (R-88 Part 2, agent >= v0.105.0) says WHY AgeSecs is nil: "absent" (a positive
|
||||
// determination that no backup has ever landed) or "unknown" (the agent could not tell —
|
||||
// unreadable storage, unparseable timestamp). "known" accompanies a real age.
|
||||
//
|
||||
// EMPTY MEANS LEGACY — an agent older than v0.105.0 simply omits the field. It does NOT mean
|
||||
// "unknown", and the distinction is load-bearing: see quiesce.ageStateFromWire. Never
|
||||
// discriminate on Reason instead; those strings are operator copy and will drift.
|
||||
AgeState string `json:"age_state"`
|
||||
}
|
||||
|
||||
// BackupResponse mirrors the agent's POST /backup payload.
|
||||
|
||||
@@ -29,6 +29,12 @@ const FeatureNetstorageVerify Feature = "netstorage_verify"
|
||||
// (GET/POST /guest/memory) shipped together, so GET /guest/memory IS the capability signal.
|
||||
const FeatureGuestMemoryResize Feature = "guest_memory_resize"
|
||||
|
||||
// FeatureBackupAgeState is R-88 Part 2 (agent v0.105.0): GET /backup/due carries `age_state`,
|
||||
// distinguishing "never backed up" (absent) from "could not tell" (unknown). There is no route
|
||||
// probe for it — the signal is a FIELD on an existing route, so the version floor is the gate and
|
||||
// an empty field means legacy.
|
||||
const FeatureBackupAgeState Feature = "backup_age_state"
|
||||
|
||||
// SupportState is a probe verdict. The zero value is SupportUnknown (fail-open: unknown never
|
||||
// refuses — the existing agent-error paths speak honestly when the agent is down).
|
||||
type SupportState int
|
||||
@@ -99,6 +105,8 @@ var errNoMemoryProbe = errors.New("agentapi: prober does not support the guest-m
|
||||
var featureMinAgent = map[Feature]string{
|
||||
FeatureNetstorageVerify: "0.81.0",
|
||||
FeatureGuestMemoryResize: "0.90.0",
|
||||
// R-88 Part 2: /backup/due carries age_state, distinguishing "never backed up" from "cannot tell".
|
||||
FeatureBackupAgeState: "0.105.0",
|
||||
}
|
||||
|
||||
// AgentVersionReporter is optionally implemented by a SupportProber (*Client is one): it reports
|
||||
|
||||
@@ -791,8 +791,11 @@ func statusRank(status string) int {
|
||||
// — while it is still retrying behind the R-88 breaker. A customer can take no action on a failed
|
||||
// whole-guest backup; that is the same harm R-97b removes, re-introduced through the front door.
|
||||
//
|
||||
// These follow the R-85 precedent exactly: a type in the hub's `allowedEventTypes` with NO
|
||||
// `customerMessages` entry, so the dispatcher structurally cannot route it to a customer.
|
||||
// Operator-only is enforced hub-side by `notify.operatorOnlyEvents` (hub >= v0.79.0, R-97c), NOT by
|
||||
// the absence of a `customerMessages` entry — v0.177.0 claimed the latter and was WRONG: the hub
|
||||
// falls back to the raw message when the entry is missing, and the only customer gate is
|
||||
// `prefs.EnabledEvents`, which is configuration. Adding a type to the allowlist does NOT make it
|
||||
// operator-only; it must go in that register too.
|
||||
//
|
||||
// HUB DEPENDENCY: both types MUST be present in the hub's allowedEventTypes or POST /event 400s
|
||||
// (the recorded allowlist gotcha). Do not deploy this controller ahead of that hub change.
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package quiesce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-88 Part 2 (controller half) — a nil age is no longer self-licensing.
|
||||
//
|
||||
// The valve fires only on a POSITIVE claim of "never backed up". These tests assert BEHAVIOUR (were
|
||||
// stacks stopped?), never a log line — a controller that logs the right thing and then does the
|
||||
// wrong thing must fail here.
|
||||
|
||||
// stateBackend is a tierBackend whose DueFor also reports an age_state wire string.
|
||||
type stateBackend struct {
|
||||
*tierBackend
|
||||
wire map[string]string // target → age_state as sent by the agent ("" = legacy)
|
||||
}
|
||||
|
||||
func (b *stateBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
||||
due, age, _, err := b.tierBackend.DueFor(ctx, target)
|
||||
return due, age, b.wire[target], err
|
||||
}
|
||||
|
||||
func stateLoop(t *testing.T, wire map[string]string, st *fakeStacks, logTo *strings.Builder) *Loop {
|
||||
t.Helper()
|
||||
be := newTierBackend()
|
||||
be.tiers = []BackupTier{{Target: "local"}}
|
||||
be.dueSet["local"] = true
|
||||
be.phases["local"] = []string{phaseDone}
|
||||
sb := &stateBackend{tierBackend: be, wire: wire}
|
||||
l := windowLoop(t, sb, st, "02:30", atBudapest(12, 0)) // 12:00 — firmly OUTSIDE [04:30, 08:30)
|
||||
if logTo != nil {
|
||||
l.logger = log.New(logTo, "", 0)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// ── SCENARIO A — UNKNOWN does not bypass the window ──────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make valveLicensed return true for AgeStateUnknown (the pre-fix
|
||||
// behaviour, where any nil age fired the valve) and this fails with
|
||||
//
|
||||
// "R-88 Part 2: an UNKNOWN age bypassed the backup window and stopped 1 stack(s) — an unreadable
|
||||
// storage must not masquerade as a first-ever backup"
|
||||
//
|
||||
// Restored.
|
||||
func TestAgeState_UnknownDoesNotBypassTheWindow(t *testing.T) {
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := stateLoop(t, map[string]string{"local": "unknown"}, st, nil)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(st.stoppedNames()); got != 0 {
|
||||
t.Fatalf("R-88 Part 2: an UNKNOWN age bypassed the backup window and stopped %d stack(s) — "+
|
||||
"an unreadable storage must not masquerade as a first-ever backup", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — ABSENT still runs outside the window ────────────────────────────────────────
|
||||
//
|
||||
// B is what makes A safe. An implementation that never licensed the valve would pass A and silently
|
||||
// starve every new box.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop AgeStateAbsent from valveLicensed (keeping only legacy) and
|
||||
// this fails with
|
||||
//
|
||||
// "a genuine first-ever backup (absent) must RUN outside the window; 0 stack(s) stopped — the
|
||||
// safety valve was lost and a new box would starve"
|
||||
//
|
||||
// Restored.
|
||||
func TestAgeState_AbsentStillRunsOutsideTheWindow(t *testing.T) {
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := stateLoop(t, map[string]string{"local": "absent"}, st, nil)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(st.stoppedNames()) == 0 {
|
||||
t.Fatal("a genuine first-ever backup (absent) must RUN outside the window; 0 stack(s) stopped — " +
|
||||
"the safety valve was lost and a new box would starve")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — old agent, new controller: TODAY'S behaviour exactly ────────────────────────
|
||||
//
|
||||
// Asserts BEHAVIOUR, not the degrade log line: a controller that logs the degrade and then defers
|
||||
// would pass a log-only assertion while silently changing behaviour on every un-upgraded box.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop AgeStateLegacy from valveLicensed (treating a missing field
|
||||
// as unknown — the "safer-looking" choice) and this fails with
|
||||
//
|
||||
// "C: a pre-v0.105.0 agent must behave EXACTLY as before — nil age fires the valve. 0 stack(s)
|
||||
// stopped; an un-upgraded box just silently stopped backing up outside its window"
|
||||
//
|
||||
// Restored.
|
||||
func TestAgeState_LegacyAgentKeepsTodaysBehaviour(t *testing.T) {
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
var logbuf strings.Builder
|
||||
l := stateLoop(t, map[string]string{"local": ""}, st, &logbuf) // NO field on the wire
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(st.stoppedNames()) == 0 {
|
||||
t.Fatal("C: a pre-v0.105.0 agent must behave EXACTLY as before — nil age fires the valve. " +
|
||||
"0 stack(s) stopped; an un-upgraded box just silently stopped backing up outside its window")
|
||||
}
|
||||
// ...and the degrade must be VISIBLE, or a fleet drifts without anyone knowing.
|
||||
if !strings.Contains(logbuf.String(), "age_state") {
|
||||
t.Fatalf("the legacy degrade must be logged once; log:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The degrade is logged ONCE, not every poll.
|
||||
func TestAgeState_LegacyDegradeLoggedOnce(t *testing.T) {
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
var logbuf strings.Builder
|
||||
l := stateLoop(t, map[string]string{"local": ""}, st, &logbuf)
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(logbuf.String(), "pre-v0.105.0"); n != 1 {
|
||||
t.Fatalf("the legacy degrade must be logged ONCE per process, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// An unrecognised FUTURE state maps to legacy, not to unknown — a newer agent inventing a fourth
|
||||
// value must not accidentally acquire "unknown" semantics from a controller that never heard of it.
|
||||
func TestAgeState_UnrecognisedWireValueIsLegacy(t *testing.T) {
|
||||
for _, wire := range []string{"", "known", "absent", "unknown", "quantum", "TRUE", "0"} {
|
||||
got := ageStateFromWire(wire)
|
||||
switch wire {
|
||||
case "known", "absent", "unknown":
|
||||
if string(got) != wire {
|
||||
t.Errorf("%q must map to itself, got %q", wire, got)
|
||||
}
|
||||
default:
|
||||
if got != AgeStateLegacy {
|
||||
t.Errorf("%q must map to LEGACY (fail toward known behaviour), got %q", wire, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// valveLicensed as a truth table — the contract, independent of the loop.
|
||||
func TestAgeState_ValveLicenceTable(t *testing.T) {
|
||||
age := int64(3600)
|
||||
cases := []struct {
|
||||
name string
|
||||
t dueTier
|
||||
want bool
|
||||
}{
|
||||
{"absent licenses", dueTier{state: AgeStateAbsent}, true},
|
||||
{"legacy licenses (un-upgraded agent keeps old behaviour)", dueTier{state: AgeStateLegacy}, true},
|
||||
{"unknown does NOT license", dueTier{state: AgeStateUnknown}, false},
|
||||
{"known with a real age needs no licence", dueTier{state: AgeStateKnown, ageSecs: &age}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := valveLicensed([]dueTier{c.t}); got != c.want {
|
||||
t.Errorf("%s: valveLicensed = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
// One unknown tier must not be licensed by a sibling that is merely known-with-age.
|
||||
if valveLicensed([]dueTier{{state: AgeStateUnknown}, {state: AgeStateKnown, ageSecs: &age}}) {
|
||||
t.Error("a known sibling must not license an unknown tier's valve")
|
||||
}
|
||||
}
|
||||
@@ -33,9 +33,10 @@ type agedBackend struct {
|
||||
ages map[string]*int64
|
||||
}
|
||||
|
||||
func (a *agedBackend) DueFor(ctx context.Context, target string) (bool, *int64, error) {
|
||||
due, _, err := a.tierBackend.DueFor(ctx, target)
|
||||
return due, a.ages[target], err
|
||||
func (a *agedBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
||||
due, _, _, err := a.tierBackend.DueFor(ctx, target)
|
||||
// A real age implies a KNOWN state — that is what the agent would send.
|
||||
return due, a.ages[target], string(AgeStateKnown), err
|
||||
}
|
||||
|
||||
// SCENARIO D — a genuinely never-backed-up box still gets its first backup, outside the window.
|
||||
@@ -114,7 +115,7 @@ func TestContract_SafetyValveBoundary(t *testing.T) {
|
||||
{"past cadence+24h → the valve fires", h(49), true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := scheduledRunAllowed(outside, window, c.age, cadence24); got != c.want {
|
||||
if got := scheduledRunAllowed(outside, window, c.age, true, cadence24); got != c.want {
|
||||
t.Errorf("CONTRACT VIOLATED: %s → scheduledRunAllowed = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,6 +100,8 @@ type Loop struct {
|
||||
mu sync.Mutex
|
||||
// degradeOnce reports the pre-R-82 agent fallback exactly once per process (see tiers.go).
|
||||
degradeOnce sync.Once
|
||||
// ageStateDegradeOnce reports a pre-v0.105.0 agent (no age_state) exactly once (R-88 Part 2).
|
||||
ageStateDegradeOnce sync.Once
|
||||
// breaker (R-88) defers the QUIESCE for a tier whose backups keep failing, so a broken target
|
||||
// cannot stop the customer's apps every 5 minutes forever. Scheduled path only — see breaker.go.
|
||||
breaker *failureBreaker
|
||||
@@ -224,7 +226,7 @@ func (l *Loop) runOnce(ctx context.Context) error {
|
||||
// cadence+24h" — cannot be suppressed by a fresher sibling tier.
|
||||
if l.windowStartFn != nil {
|
||||
window := l.windowStartFn()
|
||||
if !scheduledRunAllowed(l.now().In(budapestLocation()), window, oldestAge(dueTiers), l.cadence) {
|
||||
if !scheduledRunAllowed(l.now().In(budapestLocation()), window, oldestAge(dueTiers), valveLicensed(dueTiers), l.cadence) {
|
||||
from, to := gateBounds(window)
|
||||
l.logger.Printf("[DEBUG] [quiesce] scheduled backup due but outside the backup window [%s–%s) — deferring to the next poll inside it", from, to)
|
||||
return nil
|
||||
@@ -288,6 +290,31 @@ func (l *Loop) dropBackedOffTiers(tiers []dueTier) []dueTier {
|
||||
return kept
|
||||
}
|
||||
|
||||
// valveLicensed reports whether ANY due tier holds a POSITIVE claim of "never backed up" — the only
|
||||
// thing that may fire the window-gate safety valve on a nil age (R-88 Part 2).
|
||||
//
|
||||
// Two states license it, and the second is the important one:
|
||||
// - AgeStateAbsent — the agent looked and there is genuinely nothing there;
|
||||
// - AgeStateLegacy — a pre-v0.105.0 agent that cannot tell us either way. Preserving the OLD
|
||||
// behaviour is correct here: reading its silence as "unknown" would stop the valve firing on
|
||||
// every un-upgraded box, so a genuinely new box would never take its first backup outside its
|
||||
// window and nobody would notice for weeks. The MinAgent floor drives the upgrade; the valve is
|
||||
// not the place to force it.
|
||||
//
|
||||
// AgeStateUnknown does NOT license it. That is the entire fix: an unreadable storage no longer
|
||||
// masquerades as a first-ever backup.
|
||||
func valveLicensed(tiers []dueTier) bool {
|
||||
for _, t := range tiers {
|
||||
if t.ageSecs != nil {
|
||||
continue // a real age needs no licence; the age comparison decides
|
||||
}
|
||||
if t.state == AgeStateAbsent || t.state == AgeStateLegacy {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// oldestAge returns the largest (most overdue) age among the due tiers; nil when any tier has never
|
||||
// backed up (nil age = "never", which is maximally overdue and must win).
|
||||
func oldestAge(tiers []dueTier) *int64 {
|
||||
@@ -569,26 +596,28 @@ const (
|
||||
// absence of a signal was read as a specific value, and each time the fix was the same shape:
|
||||
// give "unknown" its own representation instead of letting it collapse into a real answer.
|
||||
//
|
||||
// ── WHAT IS AND IS NOT FIXED HERE ────────────────────────────────────────────────────────────
|
||||
// ── CLOSED BY R-88 PART 2 (agent v0.105.0 + controller v0.178.0) ─────────────────────────────
|
||||
//
|
||||
// The nil branch below STILL fires the valve, and that is currently correct-by-necessity, not by
|
||||
// design: the controller cannot yet tell the two apart. The agent's `/backup/due` returns
|
||||
// BYTE-IDENTICAL responses for "the storage read errored" and "there has genuinely never been a
|
||||
// backup" — same `Due: true`, same `Reason: "no successful backup recorded yet"`, same nil
|
||||
// `AgeSecs`. The root cause is agent-side: `newestArchiveOn` (localapi/server.go) documents that
|
||||
// errors "degrade to unknown, never to no-backup", but its `(time.Time, bool)` signature cannot
|
||||
// represent unknown, so the error collapses into a positive claim of "never".
|
||||
// The value is now THREE-STATE, not merely "nil or not". The agent reports `age_state` on
|
||||
// /backup/due — `known` / `absent` / `unknown` — and a nil age fires the valve only when
|
||||
// `valveLicensed` finds a tier holding a POSITIVE claim of "never backed up".
|
||||
//
|
||||
// Distinguishing them needs a new field on `/backup/due` plus a compat rule in both directions →
|
||||
// tracked as its own task (R-88 Part 2, agent-side). Until then the R-88 BREAKER is what bounds the
|
||||
// damage: an unknown-driven cycle may still run once outside the window, but it can no longer repeat
|
||||
// every 5 minutes.
|
||||
// It used to be that the agent returned BYTE-IDENTICAL responses for "the storage read errored" and
|
||||
// "there has genuinely never been a backup" (same Due, same Reason, same nil AgeSecs), because
|
||||
// `newestArchiveOn`'s `(time.Time, bool)` signature could not represent "unknown" — while its own
|
||||
// doc comment promised exactly that. An unreadable storage therefore masqueraded as a first-ever
|
||||
// backup and quiesced customer apps outside the window. Fixed at the source.
|
||||
//
|
||||
// DO NOT "fix" this by deleting the nil branch. Scenario D — a genuinely never-backed-up box that is
|
||||
// only ever powered on outside its window — depends on it, and TestContract_NeverBackedUp_RunsOutside
|
||||
// -TheWindow will fail if you do. Silencing the valve would trade a loud bug for a silent one: a box
|
||||
// that never backs up at all, with nobody noticing for weeks.
|
||||
func scheduledRunAllowed(now time.Time, windowStart string, lastAgeSecs *int64, cadence time.Duration) bool {
|
||||
// STILL LICENSED, DELIBERATELY: `AgeStateLegacy` — a pre-v0.105.0 agent that omits the field. Its
|
||||
// silence must NOT be read as "unknown", or the valve stops firing on every un-upgraded box and a
|
||||
// genuinely new box never takes its first backup. The MinAgent floor drives the upgrade instead.
|
||||
//
|
||||
// DO NOT "fix" this by deleting the nil branch, or by dropping the legacy case from valveLicensed.
|
||||
// Scenario D — a genuinely never-backed-up box that is only ever powered on outside its window —
|
||||
// depends on BOTH, and TestContract_NeverBackedUp_RunsOutsideTheWindow will fail if you do.
|
||||
// Silencing the valve trades a loud bug for a silent one: a box that never backs up at all, with
|
||||
// nobody noticing for weeks.
|
||||
func scheduledRunAllowed(now time.Time, windowStart string, lastAgeSecs *int64, valveOK bool, cadence time.Duration) bool {
|
||||
startMin, err := backupwindow.ParseHHMM(windowStart)
|
||||
if err != nil {
|
||||
return true
|
||||
@@ -599,7 +628,10 @@ func scheduledRunAllowed(now time.Time, windowStart string, lastAgeSecs *int64,
|
||||
}
|
||||
// Outside the window: only the safety valve may run it.
|
||||
if lastAgeSecs == nil {
|
||||
return true // no recorded backup yet — never withhold the first one
|
||||
// R-88 Part 2: a nil age is no longer self-licensing. It fires the valve ONLY on a positive
|
||||
// "never backed up" (or a legacy agent that cannot say). An UNKNOWN age — an unreadable
|
||||
// storage — now defers, which is the whole point of this arc.
|
||||
return valveOK
|
||||
}
|
||||
return time.Duration(*lastAgeSecs)*time.Second > cadence+24*time.Hour
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestScheduledRunAllowed(t *testing.T) {
|
||||
{"unparseable window fails open", atBudapest(12, 0), "nonsense", h(20), true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := scheduledRunAllowed(c.now, c.window, c.age, cadence24); got != c.want {
|
||||
if got := scheduledRunAllowed(c.now, c.window, c.age, true, cadence24); got != c.want {
|
||||
t.Errorf("%s: scheduledRunAllowed(%s, %q, age, cadence) = %v, want %v",
|
||||
c.name, c.now.Format("15:04"), c.window, got, c.want)
|
||||
}
|
||||
|
||||
@@ -46,15 +46,54 @@ type TieredBackend interface {
|
||||
Backend
|
||||
// Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent.
|
||||
Tiers(ctx context.Context) ([]BackupTier, error)
|
||||
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, err error)
|
||||
// DueFor returns due-ness, the age (nil when unavailable) and the R-88 Part 2 age STATE as the
|
||||
// agent sent it ("" = legacy agent, which is NOT the same as "unknown").
|
||||
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, ageState string, err error)
|
||||
StartBackupFor(ctx context.Context, target string) (jobID string, err error)
|
||||
BackupStatusFor(ctx context.Context, target string) (phase string, err error)
|
||||
}
|
||||
|
||||
// dueTier is a tier this cycle must back up.
|
||||
// AgeState (R-88 Part 2) is why a tier's age is nil. It mirrors the agent's `age_state` wire field.
|
||||
//
|
||||
// THE ZERO VALUE IS LEGACY, NOT UNKNOWN, and that is the whole point of the type. An agent older than
|
||||
// v0.105.0 omits the field entirely; reading that silence as "unknown" would stop the controller
|
||||
// firing its first-backup safety valve on un-upgraded boxes, so a genuinely new box would never back
|
||||
// up outside its window and nobody would notice for weeks. Preserving the KNOWN behaviour is correct;
|
||||
// the MinAgent floor is what drives the upgrade.
|
||||
type AgeState string
|
||||
|
||||
const (
|
||||
// AgeStateLegacy — the agent did not send the field. Behave exactly as before R-88 Part 2.
|
||||
AgeStateLegacy AgeState = ""
|
||||
// AgeStateKnown — the age is real.
|
||||
AgeStateKnown AgeState = "known"
|
||||
// AgeStateAbsent — a POSITIVE determination of "never backed up". The ONLY state (besides legacy)
|
||||
// that may fire the window-gate safety valve.
|
||||
AgeStateAbsent AgeState = "absent"
|
||||
// AgeStateUnknown — the agent could not tell. Still due, but it must NOT bypass the window gate.
|
||||
AgeStateUnknown AgeState = "unknown"
|
||||
)
|
||||
|
||||
// ageStateFromWire maps the agent's string to an AgeState, mapping anything unrecognised to LEGACY.
|
||||
//
|
||||
// An unknown FUTURE value is treated as legacy on purpose: a newer agent inventing a fourth state
|
||||
// must not accidentally acquire "unknown" semantics from a controller that has never heard of it.
|
||||
// Fail toward the behaviour we already understand.
|
||||
func ageStateFromWire(s string) AgeState {
|
||||
switch AgeState(s) {
|
||||
case AgeStateKnown, AgeStateAbsent, AgeStateUnknown:
|
||||
return AgeState(s)
|
||||
default:
|
||||
return AgeStateLegacy
|
||||
}
|
||||
}
|
||||
|
||||
type dueTier struct {
|
||||
target string // "" = the untargeted single-tier path (pre-R-82 agent)
|
||||
ageSecs *int64
|
||||
// state (R-88 Part 2) disambiguates a nil ageSecs. Empty = legacy agent.
|
||||
state AgeState
|
||||
}
|
||||
|
||||
// resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place.
|
||||
@@ -88,14 +127,20 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo
|
||||
return l.resolveUntargeted(ctx)
|
||||
}
|
||||
for _, t := range tiers {
|
||||
isDue, age, derr := tb.DueFor(ctx, t.Target)
|
||||
isDue, age, wireState, derr := tb.DueFor(ctx, t.Target)
|
||||
if derr != nil {
|
||||
// One tier's due-check failing must not silently drop the OTHER tier's backup.
|
||||
l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr)
|
||||
continue
|
||||
}
|
||||
if isDue {
|
||||
due = append(due, dueTier{target: t.Target, ageSecs: age})
|
||||
st := ageStateFromWire(wireState)
|
||||
if st == AgeStateLegacy {
|
||||
// A pre-v0.105.0 agent. Say so ONCE — the same shape as the pre-R-82 tier degrade,
|
||||
// because a silent behaviour difference between boxes is how a fleet drifts unnoticed.
|
||||
l.logAgeStateDegradeOnce()
|
||||
}
|
||||
due = append(due, dueTier{target: t.Target, ageSecs: age, state: st})
|
||||
}
|
||||
}
|
||||
return due, false, nil
|
||||
@@ -113,6 +158,22 @@ func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) {
|
||||
return []dueTier{{target: "", ageSecs: age}}, true, nil
|
||||
}
|
||||
|
||||
// logAgeStateDegradeOnce reports a pre-v0.105.0 agent exactly once per process (R-88 Part 2), the
|
||||
// same shape and for the same reason as logTierDegradeOnce: a rollout is a steady state, so logging
|
||||
// every poll would bury it, but logging zero times makes a real behaviour difference between boxes
|
||||
// invisible.
|
||||
//
|
||||
// The behaviour on such an agent is DELIBERATELY today's: a nil age still fires the window-gate
|
||||
// safety valve. Treating the missing field as "unknown" would look safer and would regress the
|
||||
// first-backup guarantee on every un-upgraded box.
|
||||
func (l *Loop) logAgeStateDegradeOnce() {
|
||||
l.ageStateDegradeOnce.Do(func() {
|
||||
l.logger.Printf("[WARN] [quiesce] agent does not report backup age_state (pre-v0.105.0) — " +
|
||||
"an unreadable storage cannot be told apart from 'never backed up', so a nil age still " +
|
||||
"bypasses the backup window as before. Upgrade the agent to close R-88.")
|
||||
})
|
||||
}
|
||||
|
||||
// logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a
|
||||
// steady state during a rollout and would otherwise log every poll; but never zero times, because a
|
||||
// silent degrade is indistinguishable from multi-tier working.
|
||||
|
||||
@@ -28,15 +28,15 @@ import (
|
||||
// tierBackend is a multi-tier fake agent. phases[target] is the phase sequence returned by
|
||||
// successive BackupStatusFor calls for that tier.
|
||||
type tierBackend struct {
|
||||
mu sync.Mutex
|
||||
tiers []BackupTier
|
||||
tiersErr error
|
||||
dueSet map[string]bool
|
||||
phases map[string][]string
|
||||
phaseIdx map[string]int
|
||||
started []string // targets StartBackupFor/StartBackup was called with, in order
|
||||
mu sync.Mutex
|
||||
tiers []BackupTier
|
||||
tiersErr error
|
||||
dueSet map[string]bool
|
||||
phases map[string][]string
|
||||
phaseIdx map[string]int
|
||||
started []string // targets StartBackupFor/StartBackup was called with, in order
|
||||
untargetedDue bool
|
||||
startErrOn string
|
||||
startErrOn string
|
||||
// stacks (optional) lets a start sample how many restarts have happened SO FAR — the direct
|
||||
// way to assert "the app had not resumed when this tier started".
|
||||
stacks *fakeStacks
|
||||
@@ -55,10 +55,12 @@ func (b *tierBackend) Tiers(context.Context) ([]BackupTier, error) {
|
||||
}
|
||||
return b.tiers, nil
|
||||
}
|
||||
func (b *tierBackend) DueFor(_ context.Context, target string) (bool, *int64, error) {
|
||||
// DueFor returns a nil age with an EMPTY age_state — i.e. the pre-v0.105.0 (legacy) shape, which
|
||||
// keeps every suite written before R-88 Part 2 asserting exactly the behaviour it always did.
|
||||
func (b *tierBackend) DueFor(_ context.Context, target string) (bool, *int64, string, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.dueSet[target], nil, nil
|
||||
return b.dueSet[target], nil, "", nil
|
||||
}
|
||||
func (b *tierBackend) StartBackupFor(_ context.Context, target string) (string, error) {
|
||||
b.mu.Lock()
|
||||
@@ -360,9 +362,9 @@ type dueErrBackend struct {
|
||||
errOn string
|
||||
}
|
||||
|
||||
func (d *dueErrBackend) DueFor(ctx context.Context, target string) (bool, *int64, error) {
|
||||
func (d *dueErrBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
||||
if target == d.errOn {
|
||||
return false, nil, fmt.Errorf("simulated due-check failure")
|
||||
return false, nil, "", fmt.Errorf("simulated due-check failure")
|
||||
}
|
||||
return d.tierBackend.DueFor(ctx, target)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user