reconcile: tier-aware restore-task deadline (S4.1 unattended offsite restore-test)
A WAN (pbs-tier) restore of a large guest exceeds the restore-task wait's 10m default → the wait expired mid-restore, teardown fired against a still-restoring (not-yet-pool-associated) scratch guest → leak + a phantom VM.Allocate 403. - RestoreTestSpec.RestoreTaskTimeout (0→10m default); the restore WaitTask passes it. Local tier unchanged (10m). - config RestoreTestPBSRestoreTimeoutSeconds + accessor (default 120m). - main restoreTaskTimeout(cfg,tier): configured PBS timeout only when tier==pbs, else 0. Both scheduler + selftest spec builds. - Tests + WaitOptions red-proof + accessor contract. The "grant scratch-band VM.Allocate" follow-up is diagnosed not blind-applied: the scratch is restored INTO /pool/felhom (ACL already grants VM.Allocate), so the earlier 403 was a consequence of the timeout. No ACL/host-install change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -198,6 +198,12 @@ type BackupConfig struct {
|
||||
// always excluded. Defaults to 990000–990009.
|
||||
ScratchVMIDMin int `json:"scratch_vmid_min"`
|
||||
ScratchVMIDMax int `json:"scratch_vmid_max"`
|
||||
// RestoreTestPBSRestoreTimeoutSeconds bounds the wait on a PBS-tier (offsite/WAN) restore-test
|
||||
// restore task; 0 → default 120m. A large guest restored over a slow home uplink runs long, and
|
||||
// for an UNATTENDED nightly test a false timeout (→ mid-restore teardown → leaked scratch) is
|
||||
// worse than a slow pass. Very large guests may need a higher value. LOCAL-tier restores keep
|
||||
// the 10m WaitOptions default (a local restore hanging 10m is a genuine fault).
|
||||
RestoreTestPBSRestoreTimeoutSeconds int `json:"restore_test_pbs_restore_timeout_seconds"`
|
||||
|
||||
// PBS (slice 6 Phase B). The verify maintenance loop runs on its own cadence (cheaper +
|
||||
// more frequent than the full restore-test); 0 → default (6h), negative → disabled.
|
||||
@@ -245,6 +251,14 @@ func (b BackupConfig) BackupCadence() time.Duration {
|
||||
return 24 * time.Hour
|
||||
}
|
||||
|
||||
// RestoreTestPBSRestoreTimeout returns the PBS-tier restore-task wait: positive as-is, else 120m.
|
||||
func (b BackupConfig) RestoreTestPBSRestoreTimeout() time.Duration {
|
||||
if b.RestoreTestPBSRestoreTimeoutSeconds > 0 {
|
||||
return time.Duration(b.RestoreTestPBSRestoreTimeoutSeconds) * time.Second
|
||||
}
|
||||
return 120 * time.Minute
|
||||
}
|
||||
|
||||
// defaultBackupTarget is the offsite PBS storage whole-guest backups land on by default. It is
|
||||
// SEPARATE HARDWARE from the guest's own disk (a PBS datastore on the DooPlex box), so a host
|
||||
// disk/hardware failure doesn't take the backups with it — that's what makes it real DR. Proven
|
||||
|
||||
@@ -5,8 +5,29 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestRestoreTestPBSRestoreTimeout mirrors the BackupCadence accessor contract: positive as-is,
|
||||
// 0 → default (120m), negative → default.
|
||||
func TestRestoreTestPBSRestoreTimeout(t *testing.T) {
|
||||
cases := []struct {
|
||||
secs int
|
||||
want time.Duration
|
||||
}{
|
||||
{0, 120 * time.Minute},
|
||||
{-5, 120 * time.Minute},
|
||||
{1800, 30 * time.Minute},
|
||||
{7200, 120 * time.Minute},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := BackupConfig{RestoreTestPBSRestoreTimeoutSeconds: c.secs}.RestoreTestPBSRestoreTimeout()
|
||||
if got != c.want {
|
||||
t.Errorf("RestoreTestPBSRestoreTimeout(secs=%d) = %v, want %v", c.secs, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedMasksSecret(t *testing.T) {
|
||||
c := Default()
|
||||
c.Proxmox.Token = "felhom-agent@pve!agent=b6547d9d-08ec-4f22-beb8-a551dc2cd69d"
|
||||
|
||||
Reference in New Issue
Block a user