controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve

MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a
licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY
agent, and refuses it for AgeStateUnknown. An unreadable storage no longer
masquerades as a first-ever backup and no longer quiesces apps outside the window.

A missing wire field means legacy, not unknown — deliberately. Treating it as
unknown would stop the valve firing on un-upgraded boxes and starve genuinely new
ones. Degrade logged once; unrecognised future values also map to legacy.

Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the
signature change compiled and vetted clean while quiesceBackend silently stopped
satisfying it — which would have degraded every box to the single-tier path with
no error. Added a compile-time witness.

Also corrects the notifier comment that claimed operator-only came from a missing
customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
This commit is contained in:
2026-07-27 18:08:56 +02:00
parent ba8bf9cd75
commit 86ea482fc1
12 changed files with 390 additions and 43 deletions
+64 -3
View File
@@ -46,15 +46,54 @@ type TieredBackend interface {
Backend
// Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent.
Tiers(ctx context.Context) ([]BackupTier, error)
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, err error)
// DueFor returns due-ness, the age (nil when unavailable) and the R-88 Part 2 age STATE as the
// agent sent it ("" = legacy agent, which is NOT the same as "unknown").
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, ageState string, err error)
StartBackupFor(ctx context.Context, target string) (jobID string, err error)
BackupStatusFor(ctx context.Context, target string) (phase string, err error)
}
// dueTier is a tier this cycle must back up.
// AgeState (R-88 Part 2) is why a tier's age is nil. It mirrors the agent's `age_state` wire field.
//
// THE ZERO VALUE IS LEGACY, NOT UNKNOWN, and that is the whole point of the type. An agent older than
// v0.105.0 omits the field entirely; reading that silence as "unknown" would stop the controller
// firing its first-backup safety valve on un-upgraded boxes, so a genuinely new box would never back
// up outside its window and nobody would notice for weeks. Preserving the KNOWN behaviour is correct;
// the MinAgent floor is what drives the upgrade.
type AgeState string
const (
// AgeStateLegacy — the agent did not send the field. Behave exactly as before R-88 Part 2.
AgeStateLegacy AgeState = ""
// AgeStateKnown — the age is real.
AgeStateKnown AgeState = "known"
// AgeStateAbsent — a POSITIVE determination of "never backed up". The ONLY state (besides legacy)
// that may fire the window-gate safety valve.
AgeStateAbsent AgeState = "absent"
// AgeStateUnknown — the agent could not tell. Still due, but it must NOT bypass the window gate.
AgeStateUnknown AgeState = "unknown"
)
// ageStateFromWire maps the agent's string to an AgeState, mapping anything unrecognised to LEGACY.
//
// An unknown FUTURE value is treated as legacy on purpose: a newer agent inventing a fourth state
// must not accidentally acquire "unknown" semantics from a controller that has never heard of it.
// Fail toward the behaviour we already understand.
func ageStateFromWire(s string) AgeState {
switch AgeState(s) {
case AgeStateKnown, AgeStateAbsent, AgeStateUnknown:
return AgeState(s)
default:
return AgeStateLegacy
}
}
type dueTier struct {
target string // "" = the untargeted single-tier path (pre-R-82 agent)
ageSecs *int64
// state (R-88 Part 2) disambiguates a nil ageSecs. Empty = legacy agent.
state AgeState
}
// resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place.
@@ -88,14 +127,20 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo
return l.resolveUntargeted(ctx)
}
for _, t := range tiers {
isDue, age, derr := tb.DueFor(ctx, t.Target)
isDue, age, wireState, derr := tb.DueFor(ctx, t.Target)
if derr != nil {
// One tier's due-check failing must not silently drop the OTHER tier's backup.
l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr)
continue
}
if isDue {
due = append(due, dueTier{target: t.Target, ageSecs: age})
st := ageStateFromWire(wireState)
if st == AgeStateLegacy {
// A pre-v0.105.0 agent. Say so ONCE — the same shape as the pre-R-82 tier degrade,
// because a silent behaviour difference between boxes is how a fleet drifts unnoticed.
l.logAgeStateDegradeOnce()
}
due = append(due, dueTier{target: t.Target, ageSecs: age, state: st})
}
}
return due, false, nil
@@ -113,6 +158,22 @@ func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) {
return []dueTier{{target: "", ageSecs: age}}, true, nil
}
// logAgeStateDegradeOnce reports a pre-v0.105.0 agent exactly once per process (R-88 Part 2), the
// same shape and for the same reason as logTierDegradeOnce: a rollout is a steady state, so logging
// every poll would bury it, but logging zero times makes a real behaviour difference between boxes
// invisible.
//
// The behaviour on such an agent is DELIBERATELY today's: a nil age still fires the window-gate
// safety valve. Treating the missing field as "unknown" would look safer and would regress the
// first-backup guarantee on every un-upgraded box.
func (l *Loop) logAgeStateDegradeOnce() {
l.ageStateDegradeOnce.Do(func() {
l.logger.Printf("[WARN] [quiesce] agent does not report backup age_state (pre-v0.105.0) — " +
"an unreadable storage cannot be told apart from 'never backed up', so a nil age still " +
"bypasses the backup window as before. Upgrade the agent to close R-88.")
})
}
// logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a
// steady state during a rollout and would otherwise log every poll; but never zero times, because a
// silent degrade is indistinguishable from multi-tier working.