R-85 Phase 2: tier rotation, persisted state, one heavy op at a time
The scheduler could only ever see cfg.Backup.BackupTarget(), so the offsite tier's archives were never candidates — which is why demo-hp's DR tier reported 'applied' with zero snapshots for five days and nobody noticed. Selection: oldest-first (operator ruling, Option 1). Never-proven sorts first, which is where the offsite tier starts. Ties break on target id so ordering is deterministic rather than following Go's randomised map order. Rotation credit only on SUCCESS — a permanently failing tier must keep sorting first, not look freshly proven and stop being retried. - backup.RestoreTestState: persisted last-success per tier (atomic tmp+rename). This genuinely needs persistence unlike R-84: R-84 had ground truth to consult (the archive is still on the storage), whereas a restore-test destroys its scratch and leaves no artifact. Corrupt/missing file -> 'nothing proven'. - backup.InFlight: host-wide one-heavy-op gate shared with the local-API backup path. A LINK concern, not a lock one — an offsite restore pulls multi-GB over the same tunnel a backup pushes one, and at ~33 MB/min both drift toward timeout, which is how a healthy tier gets recorded as failed. Callers DEFER, never cancel. - PickRestoreCandidateOn: newest archive on a named tier; '' is not an error, or every fresh box looks broken for its first week. - An empty tier is skipped and the next tried; it cannot starve, since it is still least-recently-proven once it has an archive. - POST /backup joins the gate (409 naming the holder). Red-proofs A/E/F observed with the documented text. Full suite green (29 packages, rc=0).
This commit is contained in:
Claude Code
+102
-9
@@ -32,6 +32,11 @@ type CandidatePicker func(ctx context.Context) (string, error)
|
||||
// PBS archive was classified "local" and got the 10-minute local wait.
|
||||
type SpecBuilder func(ctx context.Context, archive string) reconcile.RestoreTestSpec
|
||||
|
||||
// TierPicker resolves the newest archive on a NAMED tier, or "" when that tier holds none.
|
||||
// (*BackupRunner).PickRestoreCandidateOn satisfies it. "" must NOT be an error — a brand-new
|
||||
// offsite tier legitimately has nothing to restore yet.
|
||||
type TierPicker func(ctx context.Context, target string) (string, error)
|
||||
|
||||
// Scheduler runs the self-restore-test on an agent-internal cadence. It is the fourth daemon
|
||||
// goroutine; it does real restore→boot→destroy, so it only runs when the cadence is enabled
|
||||
// AND a valid scratch band is configured (validated by the caller before construction).
|
||||
@@ -43,6 +48,13 @@ type Scheduler struct {
|
||||
cadence time.Duration
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
// R-85 tier rotation. All optional: without them the scheduler behaves exactly as before
|
||||
// (single tier via `pick`), which keeps every existing caller and test working untouched.
|
||||
tiers []string // configured tier target ids, primary first
|
||||
tierPick TierPicker // newest archive on a named tier
|
||||
rtState *RestoreTestState // persisted last-successful-per-tier (drives oldest-first)
|
||||
inFlight *InFlight // shared with the backup path — Scenario F
|
||||
}
|
||||
|
||||
// SchedulerOptions configures a Scheduler.
|
||||
@@ -55,6 +67,14 @@ type SchedulerOptions struct {
|
||||
Spec SpecBuilder
|
||||
Cadence time.Duration // 0 → disabled
|
||||
Logger *slog.Logger
|
||||
|
||||
// R-85 (all optional — omit for the pre-R-85 single-tier behaviour):
|
||||
// Tiers are the configured tier target ids (primary first); TierPick resolves an archive on a
|
||||
// named tier; State persists last-successful-per-tier; InFlight is the shared one-heavy-op gate.
|
||||
Tiers []string
|
||||
TierPick TierPicker
|
||||
State *RestoreTestState
|
||||
InFlight *InFlight
|
||||
}
|
||||
|
||||
// NewScheduler builds a Scheduler.
|
||||
@@ -64,13 +84,17 @@ func NewScheduler(opts SchedulerOptions) *Scheduler {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Scheduler{
|
||||
runner: opts.Runner,
|
||||
pick: opts.Pick,
|
||||
store: opts.Store,
|
||||
spec: opts.Spec,
|
||||
cadence: opts.Cadence,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
runner: opts.Runner,
|
||||
pick: opts.Pick,
|
||||
store: opts.Store,
|
||||
spec: opts.Spec,
|
||||
cadence: opts.Cadence,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tiers: append([]string(nil), opts.Tiers...),
|
||||
tierPick: opts.TierPick,
|
||||
rtState: opts.State,
|
||||
inFlight: opts.InFlight,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +103,7 @@ func NewScheduler(opts SchedulerOptions) *Scheduler {
|
||||
// is heavy; the first runs one interval in) — on-demand runs use the selftest harness.
|
||||
// Returns nil on ctx cancellation.
|
||||
func (s *Scheduler) Run(ctx context.Context) error {
|
||||
if s.cadence <= 0 || s.runner == nil || s.pick == nil || s.spec == nil {
|
||||
if s.cadence <= 0 || s.runner == nil || s.spec == nil || (s.pick == nil && !s.rotating()) {
|
||||
s.logger.Info("backup: restore-test cadence disabled")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
@@ -108,7 +132,19 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
s.logger.Error("backup: restore-test has no spec builder — skipping (this is a wiring bug)")
|
||||
return
|
||||
}
|
||||
archive, err := s.pick(ctx)
|
||||
// Scenario F: join the one-heavy-operation-at-a-time gate. A restore-test PULLS a multi-GB
|
||||
// archive over the same tunnel an offsite backup PUSHES one; running both saturates the link and
|
||||
// drives each toward its timeout, which is how a healthy tier gets recorded as failed. DEFER —
|
||||
// never cancel what is already running: a deferred restore-test costs hours of coverage, a
|
||||
// cancelled backup costs the backup.
|
||||
release, busy, ok := s.inFlight.TryAcquire("restore-test")
|
||||
if !ok {
|
||||
s.logger.Info("backup: restore-test deferred — a heavy operation is already in flight", "busy", busy)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
|
||||
archive, target, err := s.pickForThisRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: restore-test could not pick a candidate; skipping", "err", err)
|
||||
return
|
||||
@@ -126,6 +162,13 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
}
|
||||
rt := ToHubRestoreTest(res, s.now())
|
||||
s.store.RecordRestoreTest(rt)
|
||||
// Rotation credit is given ONLY on success. A failing tier must keep sorting first, or a tier
|
||||
// that fails every time would look freshly proven and quietly stop being retried.
|
||||
if rt.Pass && s.rtState != nil && target != "" {
|
||||
if err := s.rtState.RecordSuccess(target, s.now()); err != nil {
|
||||
s.logger.Warn("backup: could not persist the restore-test rotation state", "target", target, "err", err)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !rt.Pass:
|
||||
// A failing restore-test is the loudest DR signal there is.
|
||||
@@ -142,3 +185,53 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// rotating reports whether multi-tier rotation is wired.
|
||||
func (s *Scheduler) rotating() bool { return len(s.tiers) > 0 && s.tierPick != nil }
|
||||
|
||||
// pickForThisRun chooses the tier and its newest archive.
|
||||
//
|
||||
// OLDEST-FIRST (operator ruling 2026-07-26, Option 1): the tier whose last SUCCESSFUL restore-test
|
||||
// is oldest goes first, never-proven first of all. Self-balancing, no config knob, and it naturally
|
||||
// prioritises a tier that has never been proven — which on this fleet was the offsite tier, unproven
|
||||
// for its entire existence while reporting `applied`.
|
||||
//
|
||||
// A tier with no archives is SKIPPED, not failed, and the next tier is tried. Skipping to a testable
|
||||
// tier is strictly better than burning the whole cadence: a brand-new offsite tier has nothing to
|
||||
// restore yet, and that is normal, not broken. It cannot starve the empty tier either — as soon as
|
||||
// it has an archive it still sorts first, because it is still the least recently proven.
|
||||
//
|
||||
// Returns ("", "", nil) when nothing anywhere is testable.
|
||||
func (s *Scheduler) pickForThisRun(ctx context.Context) (archive, target string, err error) {
|
||||
if !s.rotating() {
|
||||
a, perr := s.pick(ctx)
|
||||
return a, "", perr // pre-R-85 single-tier path; no rotation credit to record
|
||||
}
|
||||
order := s.tiers
|
||||
if s.rtState != nil {
|
||||
order = s.rtState.OldestFirst(s.tiers)
|
||||
}
|
||||
var firstErr error
|
||||
for _, t := range order {
|
||||
a, perr := s.tierPick(ctx, t)
|
||||
if perr != nil {
|
||||
// One tier's storage being unreadable must not block the others.
|
||||
s.logger.Warn("backup: restore-test candidate lookup failed for a tier; trying the next",
|
||||
"target", t, "err", perr)
|
||||
if firstErr == nil {
|
||||
firstErr = perr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if a == "" {
|
||||
s.logger.Debug("backup: restore-test tier has no archive yet; trying the next", "target", t)
|
||||
continue
|
||||
}
|
||||
s.logger.Info("backup: restore-test tier selected (oldest-proven first)", "target", t, "archive", a)
|
||||
return a, t, nil
|
||||
}
|
||||
if firstErr != nil {
|
||||
return "", "", firstErr
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user