v0.98.0 — R-82 Slice A fix: per-tier vzdump wait bound (the 30-minute false failure)
Found by live validation on demo-felhom, not by review. The first real PBS-targeted backup ran past the runner's hard-coded 30-minute WaitTask bound. The agent stopped waiting and recorded success=false WHILE THE VZDUMP KEPT RUNNING (still running 72 min later, 2.4 GB uploaded). Consequences: the tier stays permanently due, the next attempt collides with the guest lock the live vzdump holds, and the hub sees a DR tier that never succeeds — R-82's 'applied and empty' fault re-created by a timeout. Measured: ~33 MB/min over wg to Hetzner, so a first FULL ~10 GB snapshot projects to ~5h. - BackupTargetConfig.WaitTimeoutSeconds: per-tier bound. Primary 30m UNCHANGED (a local vzdump hanging 30m IS a real fault); additional tier 6h, sized from the measurement. - backup.NewBackupRunnerWithWait: per-instance (per-tier) bound. NewBackupRunner keeps its signature, so restore-test/selftest are untouched. - localapi.BackupTier.WaitTimeout: the fire-and-forget context is sized from the tier, not a fixed 2h. BOTH bounds had to move — a 6h runner bound under a 2h outer context reproduces the same false failure four hours later. Same direction as restore_test_pbs_restore_timeout_seconds: when in doubt wait LONGER. A slow backup is a slow backup; a false timeout is a corrupt status plus lock contention. Red-proof observed and restored; full suite green.
This commit is contained in:
@@ -38,21 +38,34 @@ type BackupRunner struct {
|
||||
// each successful backup, so the agent's own backups can't pile up and refill root. Empty → no prune
|
||||
// (the legacy behaviour; restore-test/selftest runners pass ""). NEVER applied to a PBS target.
|
||||
retention string
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
// waitTimeout bounds the WaitTask poll on this runner's vzdump. Per-TIER since R-82: 30m is
|
||||
// right for a local vzdump and badly wrong for an offsite PBS upload (see the 2026-07-26 live
|
||||
// failure recorded on config.BackupTargetConfig.WaitTimeoutSeconds). 0 → 30m (legacy).
|
||||
waitTimeout time.Duration
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
|
||||
// for lvm-thin); the caller may pass ModeStop for storages without snapshot support. retention is the
|
||||
// per-run prune spec ("keep-last=N", or "" to never prune) — only the periodic local backup sets it.
|
||||
func NewBackupRunner(api BackupAPI, target string, mode proxmox.BackupMode, notes, retention string, logger *slog.Logger) *BackupRunner {
|
||||
return NewBackupRunnerWithWait(api, target, mode, notes, retention, 0, logger)
|
||||
}
|
||||
|
||||
// NewBackupRunnerWithWait is NewBackupRunner plus an explicit vzdump wait bound (0 → 30m).
|
||||
func NewBackupRunnerWithWait(api BackupAPI, target string, mode proxmox.BackupMode, notes, retention string, waitTimeout time.Duration, logger *slog.Logger) *BackupRunner {
|
||||
if mode == "" {
|
||||
mode = proxmox.ModeSnapshot
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, retention: retention, logger: logger, now: func() time.Time { return time.Now().UTC() }}
|
||||
if waitTimeout <= 0 {
|
||||
waitTimeout = 30 * time.Minute
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, retention: retention,
|
||||
waitTimeout: waitTimeout, logger: logger, now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
||||
@@ -147,7 +160,7 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
||||
defer stopWatch()
|
||||
go r.watchForSnapshot(watchCtx, upid, onSnapshot)
|
||||
}
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: 30 * time.Minute}); err != nil {
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||
|
||||
Reference in New Issue
Block a user