v0.133.0: a restore-test can never fill a box's disk; leftovers retried on a timer (R-672, R-673)

Space preflight before anything is created (uncompressed size from the vzdump log / PBS snapshot, x1.2 + 5 GiB, thin metadata, off the tested guest's pool when another storage is eligible, unknown refuses, reported as a non-pass result). Failed scratch teardown and the stale-lock sweep retried every 10 min (the sweep under the heavy-op gate). A thin pool crossing 90% requests an immediate host report. Six red-proofs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-24 16:23:15 +02:00
parent d9864a94bf
commit 9bdb4dae8f
22 changed files with 1223 additions and 14 deletions
@@ -0,0 +1,176 @@
+// Package restorespace is the production seam behind reconcile.RestoreSpace (R-672, agent v0.133.0):
+// how much a restore of an archive writes, how much a storage has free, and which storages a
+// restore-test may target. Every read that cannot answer returns an error — the preflight then
+// REFUSES (reconcile/restoretest_space.go rule 3); nothing here guesses.
+package restorespace
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
+)
+
+// API is the Proxmox subset the provider reads.
+type API interface {
+	ListStorage(ctx context.Context) ([]proxmox.Storage, error)
+	NodeStorage(ctx context.Context) ([]proxmox.Storage, error)
+	StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error)
+	Permissions(ctx context.Context, aclPath string) (map[string]int, error)
+}
+
+// Provider implements reconcile.RestoreSpace.
+type Provider struct {
+	API API
+	// ThinMeta reads a thin pool's metadata-used fraction (storage.HostOps.ThinPoolMetadata).
+	ThinMeta func(ctx context.Context, vg, pool string) (float64, bool)
+	// ReadFile reads a vzdump log; nil → os.ReadFile.
+	ReadFile func(name string) ([]byte, error)
+}
+
+var _ reconcile.RestoreSpace = (*Provider)(nil)
+
+// totalWrittenRe is vzdump's own count of the bytes tar wrote into the archive — the UNCOMPRESSED size,
+// i.e. what a restore writes back ("INFO: Total bytes written: 22607360000 (22GiB, 49MiB/s)").
+var totalWrittenRe = regexp.MustCompile(`Total bytes written:\s*(\d+)`)
+
+// archiveExts are the vzdump archive suffixes; the log is the archive name without it + ".log".
+var archiveExts = []string{".tar.zst", ".tar.gz", ".tar.lzo", ".tgz", ".tar"}
+
+func (p *Provider) readFile(name string) ([]byte, error) {
+	if p.ReadFile != nil {
+		return p.ReadFile(name)
+	}
+	return os.ReadFile(name)
+}
+
+// RestoredBytes: a file-backed archive → its vzdump log's "Total bytes written"; a PBS archive → the
+// size Proxmox reports for the snapshot (its logical, uncompressed size). The archive FILE size is never
+// used: it is compressed (6.9 GB for a 22.6 GB restore, measured 2026-09-24).
+func (p *Provider) RestoredBytes(ctx context.Context, archive string) (int64, string, error) {
+	id, vol, ok := strings.Cut(archive, ":")
+	if !ok || id == "" || vol == "" {
+		return 0, "", fmt.Errorf("not a storage volid: %q", archive)
+	}
+	st, err := p.storageConfig(ctx, id)
+	if err != nil {
+		return 0, "", err
+	}
+	switch st.Type {
+	case "pbs":
+		items, err := p.API.StorageContent(ctx, id)
+		if err != nil {
+			return 0, "", fmt.Errorf("list %s: %w", id, err)
+		}
+		for _, it := range items {
+			if it.VolID == archive && it.Size > 0 {
+				return it.Size, "pbs snapshot size", nil
+			}
+		}
+		return 0, "", fmt.Errorf("archive %s not listed on %s with a size", archive, id)
+	default:
+		if st.Path == "" {
+			return 0, "", fmt.Errorf("storage %s (%s) has no path to read a vzdump log from", id, st.Type)
+		}
+		base := path.Base(vol) // "backup/vzdump-lxc-…tar.zst" → "vzdump-lxc-…tar.zst"
+		stem := ""
+		for _, ext := range archiveExts {
+			if strings.HasSuffix(base, ext) {
+				stem = strings.TrimSuffix(base, ext)
+				break
+			}
+		}
+		if stem == "" {
+			return 0, "", fmt.Errorf("unknown archive suffix: %s", base)
+		}
+		logPath := path.Join(st.Path, "dump", stem+".log")
+		b, err := p.readFile(logPath)
+		if err != nil {
+			return 0, "", fmt.Errorf("read vzdump log %s: %w", logPath, err)
+		}
+		m := totalWrittenRe.FindSubmatch(b)
+		if m == nil {
+			return 0, "", fmt.Errorf("vzdump log %s carries no \"Total bytes written\"", logPath)
+		}
+		n, err := strconv.ParseInt(string(m[1]), 10, 64)
+		if err != nil || n <= 0 {
+			return 0, "", fmt.Errorf("vzdump log %s: bad byte count %q", logPath, m[1])
+		}
+		return n, "vzdump log: total bytes written", nil
+	}
+}
+
+func (p *Provider) storageConfig(ctx context.Context, id string) (proxmox.Storage, error) {
+	all, err := p.API.ListStorage(ctx)
+	if err != nil {
+		return proxmox.Storage{}, fmt.Errorf("list storage config: %w", err)
+	}
+	for _, s := range all {
+		if s.Storage == id {
+			return s, nil
+		}
+	}
+	return proxmox.Storage{}, fmt.Errorf("storage %s not configured", id)
+}
+
+// Free reads the node's live usage for `storage`; a thin pool adds its metadata fill.
+func (p *Provider) Free(ctx context.Context, storage string) (reconcile.StorageFree, error) {
+	live, err := p.API.NodeStorage(ctx)
+	if err != nil {
+		return reconcile.StorageFree{}, fmt.Errorf("node storage: %w", err)
+	}
+	for _, s := range live {
+		if s.Storage != storage {
+			continue
+		}
+		if s.Active != 1 {
+			return reconcile.StorageFree{}, fmt.Errorf("storage %s is not active", storage)
+		}
+		fr := reconcile.StorageFree{AvailBytes: s.Avail, UsedBytes: s.Used, Thin: s.Type == "lvmthin"}
+		if fr.Thin {
+			cfg, cerr := p.storageConfig(ctx, storage)
+			if cerr == nil && p.ThinMeta != nil && cfg.VGName != "" && cfg.ThinPool != "" {
+				fr.MetaUsedFraction, fr.MetaKnown = p.ThinMeta(ctx, cfg.VGName, cfg.ThinPool)
+			}
+		}
+		return fr, nil
+	}
+	return reconcile.StorageFree{}, fmt.Errorf("storage %s not reported by the node", storage)
+}
+
+// Eligible: active, content includes `rootdir`, and the agent holds Datastore.AllocateSpace on
+// /storage/<id>. The permission is read for the SPECIFIC privilege — the box-wide grant answers every
+// path with inherited privileges (proxmox.Client.Permissions).
+func (p *Provider) Eligible(ctx context.Context) ([]string, error) {
+	live, err := p.API.NodeStorage(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("node storage: %w", err)
+	}
+	var out []string
+	for _, s := range live {
+		if s.Active != 1 || !hasContent(s.Content, "rootdir") {
+			continue
+		}
+		privs, perr := p.API.Permissions(ctx, "/storage/"+s.Storage)
+		if perr != nil || privs["Datastore.AllocateSpace"] != 1 {
+			continue
+		}
+		out = append(out, s.Storage)
+	}
+	return out, nil
+}
+
+func hasContent(list, want string) bool {
+	for _, c := range strings.Split(list, ",") {
+		if strings.TrimSpace(c) == want {
+			return true
+		}
+	}
+	return false
+}
@@ -0,0 +1,133 @@
+package restorespace
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// R-672 (v0.133.0). The provider behind the restore-test's space preflight. No test reaches a real
+// Proxmox or a real file: the API and ReadFile are fakes.
+
+type fakeAPI struct {
+	cfg     []proxmox.Storage
+	live    []proxmox.Storage
+	content map[string][]proxmox.StorageContent
+	perms   map[string]map[string]int
+}
+
+func (f fakeAPI) ListStorage(context.Context) ([]proxmox.Storage, error) { return f.cfg, nil }
+func (f fakeAPI) NodeStorage(context.Context) ([]proxmox.Storage, error) { return f.live, nil }
+func (f fakeAPI) StorageContent(_ context.Context, s string) ([]proxmox.StorageContent, error) {
+	return f.content[s], nil
+}
+func (f fakeAPI) Permissions(_ context.Context, p string) (map[string]int, error) {
+	if m, ok := f.perms[p]; ok {
+		return m, nil
+	}
+	return map[string]int{}, nil
+}
+
+const archive = "local:backup/vzdump-lxc-9201-2026_09_23-06_55_25.tar.zst"
+
+// The real log's tail (demo-hp, 2026-09-23): 22.6 GB written, a 6.91 GB archive file.
+const vzdumpLog = "2026-09-23 07:02:47 INFO: Total bytes written: 22607360000 (22GiB, 49MiB/s)\n2026-09-23 07:02:47 INFO: archive file size: 6.91GB\n"
+
+// demoHP is demo-hp's storage layout: `local` (dir, backups), `local-lvm` (thin), `nvme-scratch` (dir,
+// rootdir — but the agent holds NO grant there), a pbs.
+func demoHP() fakeAPI {
+	return fakeAPI{
+		cfg: []proxmox.Storage{
+			{Storage: "local", Type: "dir", Path: "/var/lib/vz"},
+			{Storage: "local-lvm", Type: "lvmthin", VGName: "pve", ThinPool: "data"},
+			{Storage: "nvme-scratch", Type: "dir", Path: "/mnt/hdd_1"},
+			{Storage: "felhom-pbs", Type: "pbs"},
+		},
+		live: []proxmox.Storage{
+			{Storage: "local", Type: "dir", Content: "vztmpl,backup,iso,import", Active: 1, Avail: 4 << 30, Used: 34 << 30},
+			{Storage: "local-lvm", Type: "lvmthin", Content: "images,rootdir", Active: 1, Avail: 23210892 * 1024, Used: 33277043 * 1024},
+			{Storage: "nvme-scratch", Type: "dir", Content: "images,rootdir", Active: 1, Avail: 800 << 30},
+			{Storage: "felhom-pbs", Type: "pbs", Content: "backup", Active: 0},
+		},
+		content: map[string][]proxmox.StorageContent{
+			"local":      {{VolID: archive, Size: 7417540996}},
+			"felhom-pbs": {{VolID: "felhom-pbs:backup/ct/9201/2026-09-23T02:00:00Z", Size: 21 << 30}},
+		},
+		perms: map[string]map[string]int{
+			"/storage/local":     {"Datastore.Audit": 1, "Datastore.AllocateSpace": 1},
+			"/storage/local-lvm": {"Datastore.Audit": 1, "Datastore.AllocateSpace": 1},
+			// nvme-scratch: only the inherited box-wide Datastore.Audit — the trap proxmox.Permissions names.
+			"/storage/nvme-scratch": {"Datastore.Audit": 1},
+		},
+	}
+}
+
+// TestRestoredBytes_ReadsTheUncompressedSize — the vzdump log's "Total bytes written", never the archive
+// FILE size (6.9 GB for a 22.6 GB restore).
+//
+// COMPANION RED-PROOF (REPORT): return the storage content's Size for a dir storage → 7417540996, and
+// this test fails at "the compressed file size was used".
+func TestRestoredBytes_ReadsTheUncompressedSize(t *testing.T) {
+	var asked string
+	p := &Provider{API: demoHP(), ReadFile: func(n string) ([]byte, error) { asked = n; return []byte(vzdumpLog), nil }}
+	n, src, err := p.RestoredBytes(context.Background(), archive)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n == 7417540996 {
+		t.Fatal("the compressed file size was used — the restore writes 3× that")
+	}
+	if n != 22607360000 || asked != "/var/lib/vz/dump/vzdump-lxc-9201-2026_09_23-06_55_25.log" {
+		t.Fatalf("n=%d from %q (log %q)", n, src, asked)
+	}
+}
+
+func TestRestoredBytes_UnknownIsAnError(t *testing.T) {
+	cases := map[string]*Provider{
+		"no log":           {API: demoHP(), ReadFile: func(string) ([]byte, error) { return nil, errors.New("ENOENT") }},
+		"log without size": {API: demoHP(), ReadFile: func(string) ([]byte, error) { return []byte("ERROR: failed\n"), nil }},
+	}
+	for name, p := range cases {
+		if n, _, err := p.RestoredBytes(context.Background(), archive); err == nil {
+			t.Fatalf("%s: got %d, want an error (the preflight then refuses)", name, n)
+		}
+	}
+	if _, _, err := (&Provider{API: demoHP()}).RestoredBytes(context.Background(), "local:backup/weird.vma"); err == nil {
+		t.Fatal("an unknown archive suffix must be an error")
+	}
+}
+
+func TestRestoredBytes_PBS(t *testing.T) {
+	p := &Provider{API: demoHP()}
+	n, _, err := p.RestoredBytes(context.Background(), "felhom-pbs:backup/ct/9201/2026-09-23T02:00:00Z")
+	if err != nil || n != 21<<30 {
+		t.Fatalf("n=%d err=%v", n, err)
+	}
+}
+
+// TestEligible_NeedsTheSpecificGrant — nvme-scratch takes rootdir but the agent holds only the inherited
+// Datastore.Audit there, so it is NOT eligible; `local` holds no rootdir.
+func TestEligible_NeedsTheSpecificGrant(t *testing.T) {
+	got, err := (&Provider{API: demoHP()}).Eligible(context.Background())
+	if err != nil || len(got) != 1 || got[0] != "local-lvm" {
+		t.Fatalf("eligible = %v (%v) — want only local-lvm on demo-hp", got, err)
+	}
+}
+
+func TestFree_ThinCarriesMetadata(t *testing.T) {
+	p := &Provider{API: demoHP(), ThinMeta: func(_ context.Context, vg, pool string) (float64, bool) {
+		if vg != "pve" || pool != "data" {
+			t.Fatalf("metadata read for %s/%s", vg, pool)
+		}
+		return 0.0265, true
+	}}
+	fr, err := p.Free(context.Background(), "local-lvm")
+	if err != nil || !fr.Thin || !fr.MetaKnown || fr.MetaUsedFraction != 0.0265 || fr.AvailBytes != 23210892*1024 {
+		t.Fatalf("free = %+v err=%v", fr, err)
+	}
+	if _, err := p.Free(context.Background(), "felhom-pbs"); err == nil {
+		t.Fatal("an inactive storage must be an error")
+	}
+}