controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY agent, and refuses it for AgeStateUnknown. An unreadable storage no longer masquerades as a first-ever backup and no longer quiesces apps outside the window. A missing wire field means legacy, not unknown — deliberately. Treating it as unknown would stop the valve firing on un-upgraded boxes and starve genuinely new ones. Degrade logged once; unrecognised future values also map to legacy. Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the signature change compiled and vetted clean while quiesceBackend silently stopped satisfying it — which would have degraded every box to the single-tier path with no error. Added a compile-time witness. Also corrects the notifier comment that claimed operator-only came from a missing customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
This commit is contained in:
@@ -46,15 +46,54 @@ type TieredBackend interface {
|
||||
Backend
|
||||
// Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent.
|
||||
Tiers(ctx context.Context) ([]BackupTier, error)
|
||||
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, err error)
|
||||
// DueFor returns due-ness, the age (nil when unavailable) and the R-88 Part 2 age STATE as the
|
||||
// agent sent it ("" = legacy agent, which is NOT the same as "unknown").
|
||||
DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, ageState string, err error)
|
||||
StartBackupFor(ctx context.Context, target string) (jobID string, err error)
|
||||
BackupStatusFor(ctx context.Context, target string) (phase string, err error)
|
||||
}
|
||||
|
||||
// dueTier is a tier this cycle must back up.
|
||||
// AgeState (R-88 Part 2) is why a tier's age is nil. It mirrors the agent's `age_state` wire field.
|
||||
//
|
||||
// THE ZERO VALUE IS LEGACY, NOT UNKNOWN, and that is the whole point of the type. An agent older than
|
||||
// v0.105.0 omits the field entirely; reading that silence as "unknown" would stop the controller
|
||||
// firing its first-backup safety valve on un-upgraded boxes, so a genuinely new box would never back
|
||||
// up outside its window and nobody would notice for weeks. Preserving the KNOWN behaviour is correct;
|
||||
// the MinAgent floor is what drives the upgrade.
|
||||
type AgeState string
|
||||
|
||||
const (
|
||||
// AgeStateLegacy — the agent did not send the field. Behave exactly as before R-88 Part 2.
|
||||
AgeStateLegacy AgeState = ""
|
||||
// AgeStateKnown — the age is real.
|
||||
AgeStateKnown AgeState = "known"
|
||||
// AgeStateAbsent — a POSITIVE determination of "never backed up". The ONLY state (besides legacy)
|
||||
// that may fire the window-gate safety valve.
|
||||
AgeStateAbsent AgeState = "absent"
|
||||
// AgeStateUnknown — the agent could not tell. Still due, but it must NOT bypass the window gate.
|
||||
AgeStateUnknown AgeState = "unknown"
|
||||
)
|
||||
|
||||
// ageStateFromWire maps the agent's string to an AgeState, mapping anything unrecognised to LEGACY.
|
||||
//
|
||||
// An unknown FUTURE value is treated as legacy on purpose: a newer agent inventing a fourth state
|
||||
// must not accidentally acquire "unknown" semantics from a controller that has never heard of it.
|
||||
// Fail toward the behaviour we already understand.
|
||||
func ageStateFromWire(s string) AgeState {
|
||||
switch AgeState(s) {
|
||||
case AgeStateKnown, AgeStateAbsent, AgeStateUnknown:
|
||||
return AgeState(s)
|
||||
default:
|
||||
return AgeStateLegacy
|
||||
}
|
||||
}
|
||||
|
||||
type dueTier struct {
|
||||
target string // "" = the untargeted single-tier path (pre-R-82 agent)
|
||||
ageSecs *int64
|
||||
// state (R-88 Part 2) disambiguates a nil ageSecs. Empty = legacy agent.
|
||||
state AgeState
|
||||
}
|
||||
|
||||
// resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place.
|
||||
@@ -88,14 +127,20 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo
|
||||
return l.resolveUntargeted(ctx)
|
||||
}
|
||||
for _, t := range tiers {
|
||||
isDue, age, derr := tb.DueFor(ctx, t.Target)
|
||||
isDue, age, wireState, derr := tb.DueFor(ctx, t.Target)
|
||||
if derr != nil {
|
||||
// One tier's due-check failing must not silently drop the OTHER tier's backup.
|
||||
l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr)
|
||||
continue
|
||||
}
|
||||
if isDue {
|
||||
due = append(due, dueTier{target: t.Target, ageSecs: age})
|
||||
st := ageStateFromWire(wireState)
|
||||
if st == AgeStateLegacy {
|
||||
// A pre-v0.105.0 agent. Say so ONCE — the same shape as the pre-R-82 tier degrade,
|
||||
// because a silent behaviour difference between boxes is how a fleet drifts unnoticed.
|
||||
l.logAgeStateDegradeOnce()
|
||||
}
|
||||
due = append(due, dueTier{target: t.Target, ageSecs: age, state: st})
|
||||
}
|
||||
}
|
||||
return due, false, nil
|
||||
@@ -113,6 +158,22 @@ func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) {
|
||||
return []dueTier{{target: "", ageSecs: age}}, true, nil
|
||||
}
|
||||
|
||||
// logAgeStateDegradeOnce reports a pre-v0.105.0 agent exactly once per process (R-88 Part 2), the
|
||||
// same shape and for the same reason as logTierDegradeOnce: a rollout is a steady state, so logging
|
||||
// every poll would bury it, but logging zero times makes a real behaviour difference between boxes
|
||||
// invisible.
|
||||
//
|
||||
// The behaviour on such an agent is DELIBERATELY today's: a nil age still fires the window-gate
|
||||
// safety valve. Treating the missing field as "unknown" would look safer and would regress the
|
||||
// first-backup guarantee on every un-upgraded box.
|
||||
func (l *Loop) logAgeStateDegradeOnce() {
|
||||
l.ageStateDegradeOnce.Do(func() {
|
||||
l.logger.Printf("[WARN] [quiesce] agent does not report backup age_state (pre-v0.105.0) — " +
|
||||
"an unreadable storage cannot be told apart from 'never backed up', so a nil age still " +
|
||||
"bypasses the backup window as before. Upgrade the agent to close R-88.")
|
||||
})
|
||||
}
|
||||
|
||||
// logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a
|
||||
// steady state during a rollout and would otherwise log every poll; but never zero times, because a
|
||||
// silent degrade is indistinguishable from multi-tier working.
|
||||
|
||||
Reference in New Issue
Block a user