package quiesce import ( "context" "errors" ) // R-82 Slice B — one quiesce window, two tiers. // // The agent gained per-target backup tiers in v0.97.0 ("local daily + PBS weekly"). The controller // owns quiescing, so the multi-tier schedule has to be reconciled HERE: on the weekly night both // tiers come due at once, and running two quiesce cycles would mean **two app outages for one // night's work** — which would undo the entire argument for weekly-over-daily. // // THE DEDUP RULE (specified, not emergent): // // local due | PBS due | result // ----------+---------+--------------------------------------------------------------- // yes | no | one quiesce, local backup // no | yes | one quiesce, PBS backup // yes | yes | ONE quiesce window, BOTH backups inside it — never two cycles // no | no | no quiesce // // ErrTiersUnsupported is returned by TieredBackend.Tiers when the agent does not serve // GET /backup/tiers — i.e. it predates R-82 (the endpoint 404s). It is the DESIGNED capability // probe, not an error condition: the loop degrades to the single untargeted tier and logs it once. // // It must NEVER be treated as "nothing to do". A new controller meeting an old agent must still // back up; concluding "not due" from an unrecognised response would silently stop backups // fleet-wide during a rollout — the exact failure this project has hit before (controller v0.154.0, // agent v0.91.0, the hub allowedEventTypes 400 in R-77). var ErrTiersUnsupported = errors.New("quiesce: agent does not serve /backup/tiers (pre-R-82)") // BackupTier is one tier as advertised by the agent, primary first. type BackupTier struct { Target string Primary bool } // TieredBackend is the OPTIONAL R-82 extension to Backend. A backend that does not implement it // (or whose Tiers returns ErrTiersUnsupported) drives the pre-R-82 single-tier path unchanged. // // The untargeted Backend methods are NOT redundant: they remain the single-tier path, and the agent // guarantees they keep their exact pre-R-82 meaning and response bytes. type TieredBackend interface { Backend // Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent. Tiers(ctx context.Context) ([]BackupTier, error) // DueFor returns due-ness, the age (nil when unavailable) and the R-88 Part 2 age STATE as the // agent sent it ("" = legacy agent, which is NOT the same as "unknown"). DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, ageState string, err error) StartBackupFor(ctx context.Context, target string) (jobID string, err error) BackupStatusFor(ctx context.Context, target string) (phase string, err error) } // dueTier is a tier this cycle must back up. // AgeState (R-88 Part 2) is why a tier's age is nil. It mirrors the agent's `age_state` wire field. // // THE ZERO VALUE IS LEGACY, NOT UNKNOWN, and that is the whole point of the type. An agent older than // v0.105.0 omits the field entirely; reading that silence as "unknown" would stop the controller // firing its first-backup safety valve on un-upgraded boxes, so a genuinely new box would never back // up outside its window and nobody would notice for weeks. Preserving the KNOWN behaviour is correct; // the MinAgent floor is what drives the upgrade. type AgeState string const ( // AgeStateLegacy — the agent did not send the field. Behave exactly as before R-88 Part 2. AgeStateLegacy AgeState = "" // AgeStateKnown — the age is real. AgeStateKnown AgeState = "known" // AgeStateAbsent — a POSITIVE determination of "never backed up". The ONLY state (besides legacy) // that may fire the window-gate safety valve. AgeStateAbsent AgeState = "absent" // AgeStateUnknown — the agent could not tell. Still due, but it must NOT bypass the window gate. AgeStateUnknown AgeState = "unknown" ) // ageStateFromWire maps the agent's string to an AgeState, mapping anything unrecognised to LEGACY. // // An unknown FUTURE value is treated as legacy on purpose: a newer agent inventing a fourth state // must not accidentally acquire "unknown" semantics from a controller that has never heard of it. // Fail toward the behaviour we already understand. func ageStateFromWire(s string) AgeState { switch AgeState(s) { case AgeStateKnown, AgeStateAbsent, AgeStateUnknown: return AgeState(s) default: return AgeStateLegacy } } type dueTier struct { target string // "" = the untargeted single-tier path (pre-R-82 agent) ageSecs *int64 // state (R-88 Part 2) disambiguates a nil ageSecs. Empty = legacy agent. state AgeState } // resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place. // // Returns the due tiers IN AGENT ORDER (primary first). That order is deliberate and it is a // downtime decision, not cosmetics: tiers run SEQUENTIALLY because vzdump holds a guest lock, and // the app stays stopped until the LAST tier has snapshotted. Running the fast local tier first and // the slow WAN/PBS tier last makes downtime ≈ (local backup) + (PBS snapshot); the reverse order // would make it ≈ (PBS backup) + (local snapshot), which is far worse. // // degraded is true when the agent is pre-R-82 and the caller must use the untargeted path. func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded bool, err error) { tb, ok := l.backend.(TieredBackend) if !ok { // Backend built without the tiered surface — the pre-R-82 path, no probe needed. return l.resolveUntargeted(ctx) } tiers, terr := tb.Tiers(ctx) if errors.Is(terr, ErrTiersUnsupported) { // A new controller meeting an OLD agent. Degrade — and SAY SO, once. l.logTierDegradeOnce() return l.resolveUntargeted(ctx) } if terr != nil { return nil, false, terr } if len(tiers) == 0 { // An agent that advertises no tiers cannot be backed up per-tier, but it can still be // backed up untargeted. Fail toward DOING the backup, never toward skipping it. l.logger.Printf("[WARN] [quiesce] agent advertised ZERO backup tiers — falling back to the untargeted path") return l.resolveUntargeted(ctx) } for _, t := range tiers { isDue, age, wireState, derr := tb.DueFor(ctx, t.Target) if derr != nil { // One tier's due-check failing must not silently drop the OTHER tier's backup. l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr) continue } if isDue { st := ageStateFromWire(wireState) if st == AgeStateLegacy { // A pre-v0.105.0 agent. Say so ONCE — the same shape as the pre-R-82 tier degrade, // because a silent behaviour difference between boxes is how a fleet drifts unnoticed. l.logAgeStateDegradeOnce() } due = append(due, dueTier{target: t.Target, ageSecs: age, state: st}) } } return due, false, nil } // resolveUntargeted is the pre-R-82 single-tier resolution. func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) { isDue, age, err := l.backend.Due(ctx) if err != nil { return nil, true, err } if !isDue { return nil, true, nil } return []dueTier{{target: "", ageSecs: age}}, true, nil } // logAgeStateDegradeOnce reports a pre-v0.105.0 agent exactly once per process (R-88 Part 2), the // same shape and for the same reason as logTierDegradeOnce: a rollout is a steady state, so logging // every poll would bury it, but logging zero times makes a real behaviour difference between boxes // invisible. // // The behaviour on such an agent is DELIBERATELY today's: a nil age still fires the window-gate // safety valve. Treating the missing field as "unknown" would look safer and would regress the // first-backup guarantee on every un-upgraded box. func (l *Loop) logAgeStateDegradeOnce() { l.ageStateDegradeOnce.Do(func() { l.logger.Printf("[WARN] [quiesce] agent does not report backup age_state (pre-v0.105.0) — " + "an unreadable storage cannot be told apart from 'never backed up', so a nil age still " + "bypasses the backup window as before. Upgrade the agent to close R-88.") }) } // logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a // steady state during a rollout and would otherwise log every poll; but never zero times, because a // silent degrade is indistinguishable from multi-tier working. func (l *Loop) logTierDegradeOnce() { l.degradeOnce.Do(func() { l.logger.Printf("[INFO] [quiesce] agent predates R-82 (no /backup/tiers) — using the single untargeted backup tier; per-tier scheduling is inactive until the agent is upgraded") }) } // startBackupOn starts a backup on one tier (untargeted when target is ""). func (l *Loop) startBackupOn(ctx context.Context, target string) (string, error) { if target == "" { return l.backend.StartBackup(ctx) } return l.backend.(TieredBackend).StartBackupFor(ctx, target) } // backupStatusOn reads one tier's job phase (untargeted when target is ""). func (l *Loop) backupStatusOn(ctx context.Context, target string) (string, error) { if target == "" { return l.backend.BackupStatus(ctx) } return l.backend.(TieredBackend).BackupStatusFor(ctx, target) } // tierLabel renders a tier for logs ("" → the untargeted tier). func tierLabel(target string) string { if target == "" { return "(untargeted)" } return target }