v0.133.0: a restore-test can never fill a box's disk; leftovers retried on a timer (R-672, R-673)

Space preflight before anything is created (uncompressed size from the vzdump log / PBS
snapshot, x1.2 + 5 GiB, thin metadata, off the tested guest's pool when another storage
is eligible, unknown refuses, reported as a non-pass result). Failed scratch teardown and
the stale-lock sweep retried every 10 min (the sweep under the heavy-op gate). A thin
pool crossing 90% requests an immediate host report. Six red-proofs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

internal/reconcile/restoretest_space.go

@@ -0,0 +1,176 @@
+package reconcile
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// ── The restore-test's space preflight (R-672, agent v0.133.0) ─────────────────────────────────────
+//
+// MEASURED 2026-09-24 on demo-hp: the scheduled restore-test restored 9201's archive into `local-lvm`
+// — the SAME thin pool that holds 9201 — with no free-space check. The pool reached 100 %
+// (`out_of_data_space`, `error_if_no_space`), and 9201's rootfs and data volume remounted READ-ONLY.
+// Evidence: felhom.eu `documentation/audits/night-2026-09-24/C-02…C-07`, `audits/r672-2026-09-24/`.
+//
+// THE THREE RULES, all decided here before ANY mutation (before the scratch entry is even journaled):
+//  1. SPACE FIRST. The target storage must have free data ≥ restored × factor + reserve (defaults 1.2 and
+//     5 GiB, `backup.restore_test_space_factor` / `backup.restore_test_space_reserve_gib`), and a thin
+//     pool's metadata must have room for the same share. `restored` is the UNCOMPRESSED size — the
+//     archive FILE is the wrong number: 9201's archive was 6.9 GB and its restore wrote 22.6 GB, so
+//     "file × 1.2 + 5 GiB" (14.3 GB) would have let the 2026-09-24 test run into a pool with 23 GB free.
+//  2. KEEP OFF THE TESTED GUEST'S POOL when another eligible storage (active, takes `rootdir`, and the
+//     agent holds Datastore.AllocateSpace on it) passes rule 1. With only one, rule 1 decides.
+//  3. UNKNOWN REFUSES. An unreadable size, an unreadable storage or an unknown thin-pool metadata fill is
+//     a skip with its reason, never a guess — the fail-safe direction of every guard in this project.
+// A refusal is reported to the hub as the test's RESULT ("skipped: …", pass=false), never as a pass.
+// Pinned by restoretest_space_test.go.
+
+// RestoreSpace is the preflight's seam onto the host. Production: internal/restorespace.
+type RestoreSpace interface {
+	// RestoredBytes is how many bytes restoring `archive` will write (uncompressed), and where that
+	// figure came from (for the log and the refusal).
+	RestoredBytes(ctx context.Context, archive string) (bytes int64, source string, err error)
+	// Free reports the storage's free data bytes and, for a thin pool, its metadata-used fraction.
+	Free(ctx context.Context, storage string) (StorageFree, error)
+	// Eligible lists the storages a restore-test may target: active, content `rootdir`, and the agent
+	// holds Datastore.AllocateSpace there.
+	Eligible(ctx context.Context) ([]string, error)
+}
+
+// StorageFree is one storage's free space as the preflight judges it.
+type StorageFree struct {
+	AvailBytes int64
+	UsedBytes  int64
+	Thin       bool
+	// MetaUsedFraction is the thin pool's metadata use (0..1); MetaKnown false = could not be read.
+	MetaUsedFraction float64
+	MetaKnown        bool
+}
+
+// SpacePolicy is rule 1's margin.
+type SpacePolicy struct {
+	Factor       float64 // ≥ 1
+	ReserveBytes int64
+}
+
+// DefaultSpacePolicy is 1.2 × restored + 5 GiB.
+var DefaultSpacePolicy = SpacePolicy{Factor: 1.2, ReserveBytes: 5 << 30}
+
+// SpaceVerdict is the preflight's answer.
+type SpaceVerdict struct {
+	OK       bool
+	Storage  string // the storage the restore goes to (when OK) or was judged (when not)
+	Required int64
+	Avail    int64
+	Reason   string // empty when OK
+	// Avoided is the tested guest's own storage, when rule 2 moved the restore off it.
+	Avoided string
+}
+
+// requiredBytes is rule 1's figure.
+func (p SpacePolicy) requiredBytes(restored int64) int64 {
+	f := p.Factor
+	if f < 1 {
+		f = DefaultSpacePolicy.Factor
+	}
+	return int64(float64(restored)*f) + p.ReserveBytes
+}
+
+// fits judges one storage against rule 1 (data AND thin metadata). An unknown metadata fill on a thin
+// pool refuses (rule 3).
+func fits(fr StorageFree, required int64) (bool, string) {
+	if fr.AvailBytes < required {
+		return false, fmt.Sprintf("needs %s free, has %s", gib(required), gib(fr.AvailBytes))
+	}
+	if fr.Thin {
+		if !fr.MetaKnown {
+			return false, "thin-pool metadata fill unknown"
+		}
+		// The metadata a restore of `required` bytes needs, in the pool's own proportion of metadata to
+		// data. A pool with no data yet has no proportion to read → only the absolute ceiling applies.
+		need := 0.0
+		if fr.UsedBytes > 0 {
+			need = fr.MetaUsedFraction * float64(required) / float64(fr.UsedBytes)
+		}
+		if fr.MetaUsedFraction+need > 0.9 {
+			return false, fmt.Sprintf("thin-pool metadata would reach %.0f%% (now %.0f%%)", 100*(fr.MetaUsedFraction+need), 100*fr.MetaUsedFraction)
+		}
+	}
+	return true, ""
+}
+
+func gib(b int64) string { return fmt.Sprintf("%.1f GiB", float64(b)/(1<<30)) }
+
+// sourceStorages returns the storage ids that hold the ARCHIVED guest's volumes (rootfs and every mpN
+// that names a `storage:volume`), read from the archive's own embedded config — the guest under test.
+// Bind mounts (a leading "/") carry no storage.
+func sourceStorages(rawCfg string) map[string]bool {
+	out := map[string]bool{}
+	for k, v := range archiveCurrentConfig(rawCfg) {
+		if k != "rootfs" && !(strings.HasPrefix(k, "mp") && len(k) > 2 && k[2] >= '0' && k[2] <= '9') {
+			continue
+		}
+		vol := strings.TrimSpace(strings.SplitN(strings.TrimSpace(v), ",", 2)[0])
+		if vol == "" || strings.HasPrefix(vol, "/") {
+			continue
+		}
+		if st, _, ok := strings.Cut(vol, ":"); ok && st != "" {
+			out[st] = true
+		}
+	}
+	return out
+}
+
+// PreflightRestoreSpace applies the three rules. `configured` is `backup.restore_storage`.
+func PreflightRestoreSpace(ctx context.Context, space RestoreSpace, policy SpacePolicy, archive, rawCfg, configured string) SpaceVerdict {
+	if space == nil {
+		return SpaceVerdict{Storage: configured, Reason: "no space check is wired — refusing (fail-closed)"}
+	}
+	restored, src, err := space.RestoredBytes(ctx, archive)
+	if err != nil || restored <= 0 {
+		return SpaceVerdict{Storage: configured, Reason: fmt.Sprintf("cannot tell how much the restore writes (%v)", err)}
+	}
+	required := policy.requiredBytes(restored)
+	own := sourceStorages(rawCfg)
+
+	// Rule 2: the configured storage holds the guest under test → try the others first.
+	var order []string
+	avoided := ""
+	if own[configured] {
+		eligible, eerr := space.Eligible(ctx)
+		if eerr == nil {
+			sort.Strings(eligible)
+			for _, s := range eligible {
+				if s != configured && !own[s] {
+					order = append(order, s)
+				}
+			}
+		}
+		if len(order) > 0 {
+			avoided = configured
+		}
+	}
+	order = append(order, configured)
+
+	var last SpaceVerdict
+	for _, s := range order {
+		fr, ferr := space.Free(ctx, s)
+		if ferr != nil {
+			last = SpaceVerdict{Storage: s, Required: required, Reason: fmt.Sprintf("cannot read free space on %s (%v)", s, ferr)}
+			continue
+		}
+		ok, why := fits(fr, required)
+		v := SpaceVerdict{OK: ok, Storage: s, Required: required, Avail: fr.AvailBytes}
+		if ok {
+			if s != configured {
+				v.Avoided = avoided
+			}
+			return v
+		}
+		v.Reason = fmt.Sprintf("not enough space on %s: restoring %s (%s) %s", s, gib(restored), src, why)
+		last = v
+	}
+	return last
+}