admin/felhom-controller
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.174.0 — R-82 Slice B: one quiesce window, two backup tiers

Browse Source 
MinAgent UNCHANGED — degrades gracefully against ANY older agent.

The agent gained per-target tiers in v0.97.0. The controller owns quiescing,
so the multi-tier schedule is reconciled here: every due tier is collected up
front and run inside ONE quiesce window (one stop, N sequential backups, one
resume). Two cycles on the weekly night would mean two app outages for one
night's work.

Dedup rule: local-only -> one quiesce; PBS-only -> one quiesce; BOTH due ->
ONE window with both backups inside; neither -> no quiesce.

- quiesce.TieredBackend + BackupTier + ErrTiersUnsupported (optional extension)
- agentapi: BackupTiers/BackupDueFor/StartBackupFor/BackupStatusFor;
  targetQuery("") yields an EMPTY suffix so untargeted hits the pre-R-82 route
  byte-for-byte
- Loop.resolveDueTiers = the dedup rule in one place, agent order preserved
- quiesceAndPollTiers + pollTier: app stays quiesced until the LAST tier
  snapshots (resuming earlier loses app-consistency on the DR tier). Consequence
  stated in the docs: both-due-night downtime = first tier's full backup + last
  tier's snapshot, which is why tiers run fast-first.
- Manual 'Mentes most' covers EVERY tier, due-ness ignored.
- Window-gate safety valve now uses the OLDEST due tier, so a stale DR tier
  cannot be starved by a fresher local one.

Capability detection: /backup/tiers 404 = pre-R-82 agent (the documented
route-probe mechanism). Not a featureProbes row on purpose — the loop needs the
tier LIST, not a yes/no. Degrade logged exactly once per process.

Tests +11, full suite green. Red-proofs #2 and #3 observed and restored.
This commit is contained in:
Claude Code
2026-07-26 14:40:44 +02:00
parent 47fda06ba1
commit de96efc0c5
8 changed files with 924 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controller/internal/quiesce/quiesce.go
+151 -42
View File
@@ -98,6 +98,8 @@ type Loop struct {
// two can never stop the same stacks concurrently (the persisted marker covers crash-safety across
// restarts; this covers concurrency within the process — which a manual trigger introduces).
mu sync.Mutex
// degradeOnce reports the pre-R-82 agent fallback exactly once per process (see tiers.go).
degradeOnce sync.Once
}
// New builds a Loop with sane defaults for any unset duration.
@@ -177,26 +179,47 @@ func (l *Loop) runOnce(ctx context.Context) error {
return nil
}
due, ageSecs, err := l.backend.Due(ctx)
// R-82: resolve EVERY due tier up front. This is the dedup rule (tiers.go): both tiers due on
// the weekly night yields ONE window with two backups, never two stop/start cycles.
dueTiers, _, err := l.resolveDueTiers(ctx)
if err != nil {
return fmt.Errorf("check due: %w", err)
}
if !due {
if len(dueTiers) == 0 {
return nil
}
// Window gate (Part 3) — SCHEDULED path only. TriggerNow calls quiesceAndPoll directly and is
// never gated. Disabled when no window fn is wired (pre-v0.168.0 behavior).
//
// With several tiers due, the gate is evaluated against the OLDEST (most overdue) tier's age,
// so the safety valve — "run regardless of the window once the last success is older than
// cadence+24h" — cannot be suppressed by a fresher sibling tier.
if l.windowStartFn != nil {
window := l.windowStartFn()
if !scheduledRunAllowed(l.now().In(budapestLocation()), window, ageSecs, l.cadence) {
if !scheduledRunAllowed(l.now().In(budapestLocation()), window, oldestAge(dueTiers), l.cadence) {
from, to := gateBounds(window)
l.logger.Printf("[DEBUG] [quiesce] scheduled backup due but outside the backup window [%s–%s) — deferring to the next poll inside it", from, to)
return nil
}
}
return l.quiesceAndPoll(ctx)
return l.quiesceAndPollTiers(ctx, dueTiers)
}
// oldestAge returns the largest (most overdue) age among the due tiers; nil when any tier has never
// backed up (nil age = "never", which is maximally overdue and must win).
func oldestAge(tiers []dueTier) *int64 {
var oldest *int64
for _, t := range tiers {
if t.ageSecs == nil {
return nil // never backed up — the strongest claim on the safety valve
}
if oldest == nil || *t.ageSecs > *oldest {
oldest = t.ageSecs
}
}
return oldest
}
// TriggerNow forces an app-consistent backup NOW (the manual "Mentés most" action), bypassing the
@@ -220,7 +243,10 @@ func (l *Loop) TriggerNow() error {
ctx, cancel := context.WithTimeout(context.Background(), l.maxQuiesce+5*time.Minute)
defer cancel()
l.logger.Printf("[INFO] [quiesce] manual backup requested — quiescing now")
if err := l.quiesceAndPoll(ctx); err != nil {
// Manual runs bypass due-ness (that is the point) but must still cover EVERY tier, in one
// window. A manual "Mentés most" that silently skipped the DR tier would be the same
// applied-and-empty fault in a different costume.
if err := l.quiesceAndPollTiers(ctx, l.allTiersForManualRun(ctx)); err != nil {
l.logger.Printf("[ERROR] [quiesce] manual backup cycle error: %v", err)
}
}()
@@ -232,6 +258,49 @@ func (l *Loop) TriggerNow() error {
// MUST hold l.mu. Unquiesce is guaranteed via the deferred closure (backup error, status-poll error,
// the max-quiesce bound, or context cancellation all still restart the stacks and clear the marker).
func (l *Loop) quiesceAndPoll(ctx context.Context) error {
return l.quiesceAndPollTiers(ctx, []dueTier{{target: ""}})
}
// allTiersForManualRun lists every tier a manual run should cover: all advertised tiers on an R-82
// agent, or the single untargeted tier otherwise. Due-ness is deliberately NOT consulted.
func (l *Loop) allTiersForManualRun(ctx context.Context) []dueTier {
tb, ok := l.backend.(TieredBackend)
if !ok {
return []dueTier{{target: ""}}
}
tiers, err := tb.Tiers(ctx)
if err != nil || len(tiers) == 0 {
if errors.Is(err, ErrTiersUnsupported) {
l.logTierDegradeOnce()
} else if err != nil {
l.logger.Printf("[WARN] [quiesce] manual run: tier list unavailable (%v) — using the untargeted tier", err)
}
return []dueTier{{target: ""}}
}
out := make([]dueTier, 0, len(tiers))
for _, t := range tiers {
out = append(out, dueTier{target: t.Target})
}
return out
}
// quiesceAndPollTiers is the R-82 multi-tier cycle: ONE marker, ONE stop, N backups run
// SEQUENTIALLY inside the window, ONE resume, then the tail polled to completion.
//
// Why sequential: vzdump takes a guest lock, so a second backup cannot start until the first
// finishes. Why the app stays down until the LAST tier snapshots: the whole point of quiescing is
// app-consistency, and resuming after tier 1's snapshot would leave tier 2 capturing a RUNNING app.
// Consequence, stated plainly because it is user-visible: on the both-due night downtime is
// (first tier's full backup) + (last tier's snapshot), not one snapshot. Tier ORDER therefore
// matters — see resolveDueTiers.
//
// Crash-safety is unchanged and non-negotiable: the marker is written BEFORE anything stops,
// unquiesce is guaranteed by defer, and it fires exactly once no matter which tier fails. A crash
// between two backups leaves the marker on disk and Recover() restarts the stacks at startup.
func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
if len(tiers) == 0 {
return nil
}
running := l.stacks.RunningAppStacks()
marker := Marker{Active: true, StartedAt: l.now(), StoppedStacks: running}
if err := l.writeMarker(marker); err != nil {
@@ -253,63 +322,103 @@ func (l *Loop) quiesceAndPoll(ctx context.Context) error {
}
defer unquiesce("deferred")
l.logger.Printf("[INFO] [quiesce] backup due — quiescing %d stack(s): %v", len(running), running)
for _, s := range running {
if err := l.stacks.StopStack(s); err != nil {
l.logger.Printf("[ERROR] [quiesce] stop %s: %v (continuing)", s, err)
l.logger.Printf("[INFO] [quiesce] backup due on %d tier(s) — quiescing %d stack(s): %v",
len(tiers), len(running), running)
for _, st := range running {
if err := l.stacks.StopStack(st); err != nil {
l.logger.Printf("[ERROR] [quiesce] stop %s: %v (continuing)", st, err)
}
}
jobID, err := l.backend.StartBackup(ctx)
if err != nil {
unquiesce("backup start failed")
return fmt.Errorf("start backup: %w", err)
}
marker.JobID = jobID
_ = l.writeMarker(marker) // best-effort: record the job id for diagnosis
l.logger.Printf("[INFO] [quiesce] backup job %s started — polling to completion", jobID)
deadline := l.now().Add(l.maxQuiesce)
var firstErr error
// ONE window, N tiers, sequential. The app resumes only after the LAST tier snapshots.
for i, t := range tiers {
label := tierLabel(t.target)
last := i == len(tiers)-1
jobID, err := l.startBackupOn(ctx, t.target)
if err != nil {
l.logger.Printf("[ERROR] [quiesce] start backup on tier %s: %v", label, err)
if firstErr == nil {
firstErr = fmt.Errorf("start backup on %s: %w", label, err)
}
// A tier that will not start must not hold the app down for the others.
if last {
unquiesce("last tier failed to start")
}
continue
}
marker.JobID = jobID
_ = l.writeMarker(marker) // best-effort: record the CURRENT tier's job id for diagnosis
l.logger.Printf("[INFO] [quiesce] tier %s: backup job %s started — polling", label, jobID)
phase, perr := l.pollTier(ctx, t.target, jobID, label, deadline, last, &unquiesced, unquiesce)
if perr != nil && firstErr == nil {
firstErr = perr
}
if phase == phaseFailed {
l.logger.Printf("[WARN] [quiesce] tier %s: backup job %s failed", label, jobID)
}
// The max-quiesce guard already unquiesced; keep going so the remaining tiers still run
// (the app is up — the backups simply continue without the quiesce guarantee, which is
// strictly better than skipping the DR tier entirely).
}
// Belt: if every tier failed to start, nothing above unquiesced. The deferred call covers it,
// but doing it here keeps the "resume as soon as possible" property explicit.
unquiesce("cycle complete")
return firstErr
}
// pollTier polls ONE tier's job. It returns the terminal (or snapshotted-at-deadline) phase.
//
// The app is resumed ONLY when this is the LAST tier — that is what keeps every tier
// app-consistent while still costing exactly one stop/start pair. For a non-last tier the loop
// waits for a TERMINAL phase (done/failed), because vzdump holds the guest lock and the next tier
// cannot start until this one truly finishes.
func (l *Loop) pollTier(ctx context.Context, target, jobID, label string, deadline time.Time,
last bool, unquiesced *bool, unquiesce func(string)) (string, error) {
for {
if !l.now().Before(deadline) {
l.logger.Printf("[WARN] [quiesce] max-quiesce-duration (%s) exceeded for job %s — unquiescing while the backup continues on the agent",
l.maxQuiesce, jobID)
l.logger.Printf("[WARN] [quiesce] max-quiesce-duration (%s) exceeded on tier %s (job %s) — unquiescing while the backup continues on the agent",
l.maxQuiesce, label, jobID)
unquiesce("max-quiesce guard")
return nil
return "", nil
}
phase, err := l.backend.BackupStatus(ctx)
phase, err := l.backupStatusOn(ctx, target)
if err != nil {
unquiesce("status poll failed")
return fmt.Errorf("poll backup status: %w", err)
return "", fmt.Errorf("poll backup status on %s: %w", label, err)
}
switch phase {
case phaseSnapshotted:
// 8B.2: the storage snapshot is taken — the app-stopped state is captured, so the app
// may resume NOW (downtime = until-snapshot, not until-backup-done) with no loss of
// app-consistency. unquiesce is idempotent (fires once); we then KEEP polling to
// done/failed so a new backup isn't started until this one truly finishes (and so a
// post-snapshot failure is observed). The marker is cleared on resume — a crash in this
// tail leaves the app already up, nothing to recover.
if !unquiesced {
l.logger.Printf("[INFO] [quiesce] backup job %s snapshotted — resuming app early (8B.2)", jobID)
unquiesce("snapshotted (early resume)")
// 8B.2 early resume — but ONLY on the last tier. Resuming here on a non-last tier would
// leave the following tier capturing a running app, losing app-consistency for exactly
// the DR tier we most want it on.
if last && !*unquiesced {
l.logger.Printf("[INFO] [quiesce] tier %s: job %s snapshotted — resuming app early (8B.2)", label, jobID)
unquiesce("snapshotted (early resume, last tier)")
}
case phaseDone:
// Fallback (stop/downgraded mode never emits snapshotted): resume at done, exactly 8B.
l.logger.Printf("[INFO] [quiesce] backup job %s done", jobID)
unquiesce("backup done")
return nil
if last {
l.logger.Printf("[INFO] [quiesce] tier %s: backup job %s done", label, jobID)
unquiesce("backup done")
} else {
l.logger.Printf("[INFO] [quiesce] tier %s: backup job %s done — next tier may start (app still quiesced)", label, jobID)
}
return phaseDone, nil
case phaseFailed:
// If we already resumed at snapshotted, the app is up — just note the backup failed
// (recorded for the agent's due window when it stores the failed result).
l.logger.Printf("[WARN] [quiesce] backup job %s failed", jobID)
unquiesce("backup failed")
return nil
if last {
unquiesce("backup failed")
}
return phaseFailed, nil
}
select {
case <-ctx.Done():
unquiesce("controller shutting down")
return ctx.Err()
return "", ctx.Err()
case <-time.After(l.statusPoll):
}
}