v0.133.0: a restore-test can never fill a box's disk; leftovers retried on a timer (R-672, R-673)
gates / gates (push) Successful in 13s
gates / gates (push) Successful in 13s
Space preflight before anything is created (uncompressed size from the vzdump log / PBS snapshot, x1.2 + 5 GiB, thin metadata, off the tested guest's pool when another storage is eligible, unknown refuses, reported as a non-pass result). Failed scratch teardown and the stale-lock sweep retried every 10 min (the sweep under the heavy-op gate). A thin pool crossing 90% requests an immediate host report. Six red-proofs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -50,10 +50,18 @@ type RestoreTestResult struct {
|
||||
ScratchVMID int
|
||||
Pass bool
|
||||
Verified string // "boot+running" this slice
|
||||
Skipped bool // no free scratch VMID in band → test not run
|
||||
Err error
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
Skipped bool // test not run: no free scratch VMID in band, or the space preflight refused (R-672)
|
||||
// SkipReason is set when the SPACE PREFLIGHT refused (R-672): the test did not run, and this is
|
||||
// reported to the hub as the test's result (pass=false), never as a pass. Empty for a band skip.
|
||||
SkipReason string
|
||||
// TargetStorage is where the restore went (rule 2 may move it off the tested guest's pool);
|
||||
// RequiredBytes/AvailBytes are rule 1's figures.
|
||||
TargetStorage string
|
||||
RequiredBytes int64
|
||||
AvailBytes int64
|
||||
Err error
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
// StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the
|
||||
// systemd-nesting advisory). Populated only when the start exited "WARNINGS: N";
|
||||
// always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning).
|
||||
@@ -136,6 +144,29 @@ func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) Resto
|
||||
return res
|
||||
}
|
||||
|
||||
// R-672: the space preflight, BEFORE anything is journaled or created.
|
||||
rawCfg, err := e.api.ExtractArchiveConfig(ctx, spec.Archive)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test extract archive config: %w", err)
|
||||
return res
|
||||
}
|
||||
v := PreflightRestoreSpace(ctx, e.restoreSpace, e.spacePolicy, spec.Archive, rawCfg, spec.RestoreStorage)
|
||||
res.TargetStorage, res.RequiredBytes, res.AvailBytes = v.Storage, v.Required, v.Avail
|
||||
if !v.OK {
|
||||
res.Skipped = true
|
||||
res.SkipReason = "skipped: " + v.Reason
|
||||
e.logger.Warn("restore-test SKIPPED by the space preflight (R-672) — nothing was created",
|
||||
"archive", spec.Archive, "storage", v.Storage, "required_bytes", v.Required, "avail_bytes", v.Avail, "reason", v.Reason)
|
||||
res.Duration = time.Since(now)
|
||||
return res
|
||||
}
|
||||
if v.Storage != spec.RestoreStorage {
|
||||
e.logger.Info("restore-test: restoring OFF the tested guest's own pool (R-672 rule 2)",
|
||||
"configured", spec.RestoreStorage, "avoided", v.Avoided, "target", v.Storage)
|
||||
}
|
||||
e.logger.Info("restore-test: space preflight passed", "storage", v.Storage, "required_bytes", v.Required, "avail_bytes", v.Avail)
|
||||
spec.RestoreStorage = v.Storage
|
||||
|
||||
lxc, err := e.api.ListLXC(ctx)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test list guests: %w", err)
|
||||
@@ -164,7 +195,9 @@ func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) Resto
|
||||
// Serialize on the scratch VMID's lane (inherits §10), and capture the result.
|
||||
var vmidOccupied bool
|
||||
ch := e.queue.Submit(vmid, func() error {
|
||||
vmidOccupied = e.runScratchTest(ctx, vmid, spec, &res)
|
||||
e.markScratch(vmid, true)
|
||||
defer e.markScratch(vmid, false)
|
||||
vmidOccupied = e.runScratchTest(ctx, vmid, spec, rawCfg, &res)
|
||||
return res.Err
|
||||
})
|
||||
<-ch
|
||||
@@ -182,7 +215,7 @@ func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) Resto
|
||||
// runScratchTest is the journaled body (runs on vmid's queue lane). The occupied return is true
|
||||
// ONLY when PVE synchronously refused the restore because the vmid already holds a guest (one
|
||||
// the pool-blind band scan couldn't see) — the caller then advances to the next band vmid (F2).
|
||||
func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) (occupied bool) {
|
||||
func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, rawCfg string, res *RestoreTestResult) (occupied bool) {
|
||||
base := JournalEntry{OpID: e.scratchOpID(vmid), VMID: vmid, Kind: scratchKind, Scratch: true}
|
||||
|
||||
// OWN the scratch guest's cleanup BEFORE any mutation. From here, a crash is recoverable.
|
||||
@@ -215,11 +248,7 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
|
||||
// genuinely EXTRACTED — full fidelity; the added runtime IS the verification), the two
|
||||
// structural binds → throwaway stand-ins. An unreadable archive config or an unknown
|
||||
// topology REFUSES up front — never restore a partial guest to "verify" it.
|
||||
rawCfg, err := e.api.ExtractArchiveConfig(ctx, spec.Archive)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test extract archive config: %w", err)
|
||||
return false
|
||||
}
|
||||
// The archive's config was read ONCE, by the space preflight (R-672), and is passed in.
|
||||
mountOverrides, err := drRestoreOverrides(rawCfg, spec.RestoreStorage)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user