GL-5b: restore-test full-fidelity verification (v0.76.0)
The restore-test had GL-5 finding #2's mirror image: its live-source-config bind-override path tripped PVE's drop-unlisted-mountpoints rule, so scratch guests boot-verified WITHOUT their storage mpN - weaker verification than claimed. Params now derive from the ARCHIVE's own embedded config via ExtractArchiveConfig + drRestoreOverrides (the object under test; full layout, content genuinely extracted - the added runtime IS the verification); unreadable/unknown-topology archives refuse up front. NEW mount-parity assert (2b, pre-start): restored mpN set vs the archive's - missing/mispathed/undersized/extra mpN fail the test naming the delta, so constraint (b) can never regress into a green light. MountParity + MountInventory ride the result + hub wire record (additive). Dead bindMountOverrides/archiveVMID path deleted with its tests (no reachable lookalike). DR bring-up untouched. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -308,6 +308,8 @@ func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.R
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
Warnings: res.StartWarnings,
|
||||
WarningsRecognized: res.WarningsRecognized,
|
||||
MountParity: res.MountParity,
|
||||
MountInventory: res.MountInventory,
|
||||
}
|
||||
if res.Err != nil {
|
||||
rt.Error = res.Err.Error()
|
||||
|
||||
@@ -304,6 +304,11 @@ type RestoreTest struct {
|
||||
// (⇒ false) when absent — and false is the SAFE default: the hub then treats it as an
|
||||
// unrecognized warning (louder), so a missing flag can only over-notice, never hide.
|
||||
WarningsRecognized bool `json:"warnings_recognized,omitempty"`
|
||||
// MountParity ("ok"|"mismatch"; omitted on pre-v0.76.0 agents) + MountInventory (the verified
|
||||
// mpN set) carry the GL-5b full-fidelity proof: the restored scratch matched the ARCHIVE's
|
||||
// mount layout, not just booted. Additive — a hub that predates them ignores the unknown keys.
|
||||
MountParity string `json:"mount_parity,omitempty"`
|
||||
MountInventory []string `json:"mount_inventory,omitempty"`
|
||||
}
|
||||
|
||||
// PBSSnapshot is one PBS (offsite) snapshot's inventory + integrity state (doc 03 §8, slice
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -63,6 +63,14 @@ type RestoreTestResult struct {
|
||||
// wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and
|
||||
// never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A).
|
||||
WarningsRecognized bool
|
||||
// MountParity is "ok" when the restored scratch's mpN set matches the archive's (GL-5b: the
|
||||
// assert that keeps PVE's drop-unlisted-mountpoints rule from ever regressing into a green
|
||||
// light); "mismatch" fails the test with Err naming the delta. "" on runs that never reached
|
||||
// the assert (restore failed earlier).
|
||||
MountParity string
|
||||
// MountInventory lists the parity-verified mountpoints ("mpN=<path> (<N>G)" / throwaway
|
||||
// stand-ins) — the record's proof of WHAT was verified, not just that something booted.
|
||||
MountInventory []string
|
||||
}
|
||||
|
||||
// benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start
|
||||
@@ -198,36 +206,27 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
|
||||
|
||||
// 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error
|
||||
// detection only — it does NOT make the Scratch entry terminal (teardown does).
|
||||
// A source guest with a host BIND-mount mountpoint (slice-10 data drive) can't be
|
||||
// vzrestore'd by the privsep token ("restoring 'mpN' to bind mount is only possible for
|
||||
// root"). The restore-test only needs the guest to BOOT — the data drives are irrelevant
|
||||
// (their host paths would also collide). So neutralize each source bind-mount mpN to a
|
||||
// throwaway volume on the restore storage (needs no root). Best-effort: if the source
|
||||
// config can't be read, restore as-is (a bind-mount guest then fails as before, in the verdict).
|
||||
var mountOverrides map[string]string
|
||||
if srcVMID, ok := archiveVMID(spec.Archive); ok {
|
||||
if srcCfg, cerr := e.api.GuestConfig(ctx, srcVMID); cerr == nil {
|
||||
if binds := bindMountOverrides(srcCfg.MountPoints(), spec.RestoreStorage); len(binds) > 0 {
|
||||
// PVE refuses a restore that carries mountpoint params unless `rootfs` is also set
|
||||
// ("mount points configured, but 'rootfs' not set"). Size the rootfs override from the
|
||||
// SOURCE rootfs (restore needs target >= the archive's volume). Without a parseable
|
||||
// size we can't safely override, so restore as-is (the bind-mount restore then fails
|
||||
// in the verdict rather than risking a wrong rootfs size).
|
||||
if sz := rootfsSizeGB(srcCfg.RootFS); sz > 0 {
|
||||
binds["rootfs"] = fmt.Sprintf("%s:%d", spec.RestoreStorage, sz)
|
||||
mountOverrides = binds
|
||||
e.logger.Info("restore-test: neutralizing source bind-mount mountpoints for scratch restore",
|
||||
"source_vmid", srcVMID, "scratch", vmid, "bind_mounts", len(binds)-1, "rootfs_gb", sz)
|
||||
} else {
|
||||
e.logger.Warn("restore-test: source has bind mounts but rootfs size unparseable — restoring as-is",
|
||||
"source_vmid", srcVMID, "rootfs", srcCfg.RootFS)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
e.logger.Warn("restore-test: could not read source config for mp overrides (restoring as-is)",
|
||||
"source_vmid", srcVMID, "err", cerr)
|
||||
}
|
||||
// The restore params derive from the ARCHIVE's own embedded config — the object under
|
||||
// test — exactly like DR bring-up (GL-5b closes GL-5 finding #2's mirror image): PVE's
|
||||
// explicit-params restore REQUIRES an explicit rootfs AND silently DROPS unlisted
|
||||
// mountpoints, so the pre-v0.76.0 live-source-config bind-override path boot-verified
|
||||
// scratch guests WITHOUT their storage mpN — weaker verification than it claimed. The FULL
|
||||
// layout now rides: rootfs explicit, every storage mpN passed through (its content is
|
||||
// genuinely EXTRACTED — full fidelity; the added runtime IS the verification), the two
|
||||
// structural binds → throwaway stand-ins. An unreadable archive config or an unknown
|
||||
// topology REFUSES up front — never restore a partial guest to "verify" it.
|
||||
rawCfg, err := e.api.ExtractArchiveConfig(ctx, spec.Archive)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test extract archive config: %w", err)
|
||||
return false
|
||||
}
|
||||
mountOverrides, err := drRestoreOverrides(rawCfg, spec.RestoreStorage)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test: %w", err)
|
||||
return false
|
||||
}
|
||||
e.logger.Info("restore-test: full-fidelity restore params derived from the archive config",
|
||||
"scratch", vmid, "params", len(mountOverrides))
|
||||
upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
|
||||
// Pool=DefaultPool so the scratch guest is created INTO the felhom pool — else a pool-scoped
|
||||
// token 403s on the scratch guest's config/start/destroy (SPIKE residual #2).
|
||||
@@ -262,6 +261,21 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
|
||||
res.Err = fmt.Errorf("reconcile: restore-test read scratch config: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 2b. Mount-parity assert (GL-5b, the non-hollow core): the restored guest's mpN set must
|
||||
// match the archive's — storage mpN as real volumes at their archived path+size, the two
|
||||
// structural binds as their throwaway stand-ins. This makes GL-5's PVE rule (b) —
|
||||
// unlisted mountpoints silently dropped — impossible to regress into a green light: a
|
||||
// boot-only verify cannot see a missing data volume; this can.
|
||||
inventory, delta := mountParity(archiveCurrentConfig(rawCfg), cfg.MountPoints())
|
||||
res.MountInventory = inventory
|
||||
if len(delta) > 0 {
|
||||
res.MountParity = "mismatch"
|
||||
res.Err = fmt.Errorf("reconcile: restore-test mount parity FAILED (restored scratch does not match the archive): %s", strings.Join(delta, "; "))
|
||||
return
|
||||
}
|
||||
res.MountParity = "ok"
|
||||
|
||||
for key, val := range cfg.Nets() {
|
||||
if _, err := e.api.SetConfig(ctx, vmid, map[string]string{key: withLinkDown(val)}); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test net link-down %s: %w", key, err)
|
||||
@@ -307,58 +321,72 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
|
||||
return false
|
||||
}
|
||||
|
||||
// archiveVMID extracts the source VMID from a backup archive volid. Handles PBS volids
|
||||
// ("<store>:backup/ct/<vmid>/<ts>" and the /vm/<vmid>/ form) and vzdump file volids
|
||||
// ("<store>:backup/vzdump-(lxc|qemu)-<vmid>-..."). Returns false when no vmid is found.
|
||||
func archiveVMID(volid string) (int, bool) {
|
||||
for _, re := range []*regexp.Regexp{pbsArchiveRE, vzdumpArchiveRE} {
|
||||
if m := re.FindStringSubmatch(volid); m != nil {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var (
|
||||
pbsArchiveRE = regexp.MustCompile(`/(?:ct|vm)/(\d+)/`)
|
||||
vzdumpArchiveRE = regexp.MustCompile(`vzdump-(?:lxc|qemu)-(\d+)-`)
|
||||
)
|
||||
|
||||
// bindMountOverrides maps each BIND-mount mpN (the volume part is an absolute host PATH, not a
|
||||
// "storage:volume") to a throwaway 1G volume override on restoreStorage, preserving the in-guest
|
||||
// mount path. Storage-backed mpN are left to restore normally (returns nil when there are none).
|
||||
// This is what lets the restore-test boot-verify a slice-10 enrolled guest whose data drive is a
|
||||
// host bind mount the privsep token can't otherwise restore.
|
||||
func bindMountOverrides(mps map[string]string, restoreStorage string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for key, val := range mps {
|
||||
volPart, rest, _ := strings.Cut(val, ",")
|
||||
if !strings.HasPrefix(volPart, "/") {
|
||||
continue // "storage:volume" → a real volume, not a host bind mount
|
||||
}
|
||||
mp := mountPathOf(rest)
|
||||
if mp == "" {
|
||||
mp = volPart // fall back to the host path if no explicit in-guest mp=
|
||||
}
|
||||
out[key] = throwawayVolumeOverride(restoreStorage, mp)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G
|
||||
// volume on restoreStorage at the given in-guest path, excluded from backup. Used by the
|
||||
// restore-test's source-config-derived overrides (above) and by the DR bring-up's synthesized
|
||||
// structural-bind overrides (GL-5, bringup.go — its mpN are platform constants, so it skips this
|
||||
// file's is-a-bind filtering and calls the format directly).
|
||||
// volume on restoreStorage at the given in-guest path, excluded from backup. Used by
|
||||
// drRestoreOverrides (bringup.go) for the structural binds — which both the DR bring-up and the
|
||||
// restore-test (GL-5b) derive their restore params from. (The old live-source-config
|
||||
// bindMountOverrides/archiveVMID path was deleted in v0.76.0 when the restore-test switched to
|
||||
// archive-derived params: it verified the wrong object AND tripped PVE's drop-unlisted-mountpoints
|
||||
// rule; keeping the dead lookalike reachable is how the next bug happens.)
|
||||
func throwawayVolumeOverride(restoreStorage, guestPath string) string {
|
||||
return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath)
|
||||
}
|
||||
|
||||
// mountParity compares the restored scratch guest's mpN set against the archive's current config
|
||||
// (GL-5b). Per archive mpN: a storage-backed one must exist restored at the same in-guest path
|
||||
// with at least the archived size (the pass-through recreates it at exactly that size); a
|
||||
// structural host bind must exist as its throwaway stand-in at the same path. Restored mpN slots
|
||||
// the archive doesn't carry are a delta too. Returns the verified inventory + the mismatch list
|
||||
// (empty delta = parity). Deterministic order (sorted slots).
|
||||
func mountParity(archiveCfg, restored map[string]string) (inventory, delta []string) {
|
||||
isMP := func(k string) bool {
|
||||
return len(k) > 2 && k[:2] == "mp" && k[2] >= '0' && k[2] <= '9'
|
||||
}
|
||||
slots := make([]string, 0, len(archiveCfg))
|
||||
for k := range archiveCfg {
|
||||
if isMP(k) {
|
||||
slots = append(slots, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(slots)
|
||||
for _, k := range slots {
|
||||
aval := archiveCfg[k]
|
||||
avol, arest, _ := strings.Cut(aval, ",")
|
||||
apath := mountPathOf(arest)
|
||||
if apath == "" && strings.HasPrefix(avol, "/") {
|
||||
apath = avol // a bind without an explicit mp= mounts at its host path
|
||||
}
|
||||
rval, ok := restored[k]
|
||||
if !ok {
|
||||
delta = append(delta, fmt.Sprintf("%s MISSING from the restored guest (archive: %s)", k, aval))
|
||||
continue
|
||||
}
|
||||
_, rrest, _ := strings.Cut(rval, ",")
|
||||
if rpath := mountPathOf(rrest); rpath != apath {
|
||||
delta = append(delta, fmt.Sprintf("%s path %q != archive %q", k, rpath, apath))
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(avol, "/") {
|
||||
// structural bind → its throwaway stand-in (a real volume at the same path)
|
||||
inventory = append(inventory, fmt.Sprintf("%s=%s (throwaway for the archived bind)", k, apath))
|
||||
continue
|
||||
}
|
||||
asz, rsz := rootfsSizeGB(aval), rootfsSizeGB(rval)
|
||||
if asz > 0 && rsz < asz {
|
||||
delta = append(delta, fmt.Sprintf("%s size %dG < archive %dG", k, rsz, asz))
|
||||
continue
|
||||
}
|
||||
inventory = append(inventory, fmt.Sprintf("%s=%s (%dG)", k, apath, rsz))
|
||||
}
|
||||
for k, v := range restored {
|
||||
if isMP(k) && archiveCfg[k] == "" {
|
||||
delta = append(delta, fmt.Sprintf("%s present on the restored guest but not in the archive: %s", k, v))
|
||||
}
|
||||
}
|
||||
sort.Strings(delta)
|
||||
return inventory, delta
|
||||
}
|
||||
|
||||
// mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent).
|
||||
func mountPathOf(opts string) string {
|
||||
for _, kv := range strings.Split(opts, ",") {
|
||||
|
||||
@@ -1,51 +1,13 @@
|
||||
package reconcile
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestArchiveVMID(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
volid string
|
||||
want int
|
||||
ok bool
|
||||
}{
|
||||
"pbs ct": {"felhom-pbs:backup/ct/9201/2026-06-12T18:29:58Z", 9201, true},
|
||||
"pbs vm": {"felhom-pbs:backup/vm/142/2026-06-12T18:29:58Z", 142, true},
|
||||
"vzdump lxc": {"local:backup/vzdump-lxc-9201-2026_06_12-18_29_58.tar.zst", 9201, true},
|
||||
"vzdump qemu": {"local:backup/vzdump-qemu-100-2026_06_12.vma.zst", 100, true},
|
||||
"none": {"local:iso/whatever.iso", 0, false},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, ok := archiveVMID(c.volid)
|
||||
if ok != c.ok || got != c.want {
|
||||
t.Errorf("archiveVMID(%q) = (%d,%v), want (%d,%v)", c.volid, got, ok, c.want, c.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindMountOverrides(t *testing.T) {
|
||||
mps := map[string]string{
|
||||
"mp0": "/mnt/felhom-usb/felhom-data,mp=/mnt/felhom-usb", // bind mount → override
|
||||
"mp1": "local-lvm:8,mp=/data,backup=0", // real volume → left alone
|
||||
"mp2": "/srv/extra", // bind mount, no explicit mp= → use host path
|
||||
}
|
||||
out := bindMountOverrides(mps, "local-lvm")
|
||||
if _, ok := out["mp1"]; ok {
|
||||
t.Error("mp1 is a storage volume and must NOT be overridden")
|
||||
}
|
||||
if got, want := out["mp0"], "local-lvm:1,mp=/mnt/felhom-usb,backup=0"; got != want {
|
||||
t.Errorf("mp0 override = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := out["mp2"], "local-lvm:1,mp=/srv/extra,backup=0"; got != want {
|
||||
t.Errorf("mp2 override = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// No bind mounts → nil (restore proceeds unchanged).
|
||||
if bindMountOverrides(map[string]string{"mp0": "local-lvm:8,mp=/data"}, "local-lvm") != nil {
|
||||
t.Error("expected nil when there are no bind mounts")
|
||||
}
|
||||
}
|
||||
// (TestArchiveVMID + TestBindMountOverrides were deleted with their subjects in v0.76.0/GL-5b —
|
||||
// the restore-test now derives its params from the ARCHIVE's embedded config via
|
||||
// drRestoreOverrides; the live-source-config path was the pre-fix shape.)
|
||||
|
||||
func TestRootfsSizeGB(t *testing.T) {
|
||||
cases := map[string]int{
|
||||
@@ -62,3 +24,77 @@ func TestRootfsSizeGB(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5b: the mount-parity comparator — the assert that keeps PVE's drop-unlisted-mountpoints rule
|
||||
// from regressing into a green light. Pure-function cases; the engine-level Scenario A/B live in
|
||||
// restoretest_test.go / the fake-driven tests.
|
||||
func TestMountParity(t *testing.T) {
|
||||
archive := map[string]string{
|
||||
"hostname": "demo",
|
||||
"rootfs": "local-lvm:vm-9201-disk-0,size=32G",
|
||||
"mp0": "local-lvm:vm-9201-disk-1,mp=/var/lib/docker,backup=1,size=200G",
|
||||
"mp1": "local-lvm:vm-9201-disk-2,mp=/mnt/sys_drive,backup=1,size=50G",
|
||||
"mp8": "/mnt/felhom-drives,mp=/mnt/felhom-drives",
|
||||
"mp9": "/var/lib/felhom-agent/guests/9201/bootstrap,mp=/etc/felhom-bootstrap,ro=1",
|
||||
}
|
||||
|
||||
// full parity: storage mpN at archived path+size, binds as throwaway stand-ins
|
||||
restored := map[string]string{
|
||||
"mp0": "local-lvm:vm-990000-disk-1,mp=/var/lib/docker,backup=1,size=200G",
|
||||
"mp1": "local-lvm:vm-990000-disk-2,mp=/mnt/sys_drive,backup=1,size=50G",
|
||||
"mp8": "local-lvm:vm-990000-disk-3,mp=/mnt/felhom-drives,backup=0,size=1G",
|
||||
"mp9": "local-lvm:vm-990000-disk-4,mp=/etc/felhom-bootstrap,backup=0,size=1G",
|
||||
}
|
||||
inv, delta := mountParity(archive, restored)
|
||||
if len(delta) != 0 {
|
||||
t.Fatalf("full parity expected, got delta: %v", delta)
|
||||
}
|
||||
if len(inv) != 4 {
|
||||
t.Fatalf("inventory must carry all 4 verified mpN, got %v", inv)
|
||||
}
|
||||
|
||||
// the GL-5 constraint-(b) shape: a storage mpN silently dropped → NAMED delta (Scenario B core)
|
||||
dropped := map[string]string{
|
||||
"mp1": restored["mp1"], "mp8": restored["mp8"], "mp9": restored["mp9"],
|
||||
}
|
||||
_, delta = mountParity(archive, dropped)
|
||||
if len(delta) != 1 || !strings.Contains(delta[0], "mp0 MISSING") {
|
||||
t.Fatalf("dropped mp0 must be a named delta, got %v", delta)
|
||||
}
|
||||
|
||||
// wrong path
|
||||
wrongPath := map[string]string{
|
||||
"mp0": "local-lvm:vm-990000-disk-1,mp=/elsewhere,size=200G",
|
||||
"mp1": restored["mp1"], "mp8": restored["mp8"], "mp9": restored["mp9"],
|
||||
}
|
||||
_, delta = mountParity(archive, wrongPath)
|
||||
if len(delta) != 1 || !strings.Contains(delta[0], `mp0 path "/elsewhere"`) {
|
||||
t.Fatalf("wrong path must be a named delta, got %v", delta)
|
||||
}
|
||||
|
||||
// undersized volume (a 1G stand-in where 200G of data should be = the dropped-content shape)
|
||||
small := map[string]string{
|
||||
"mp0": "local-lvm:vm-990000-disk-1,mp=/var/lib/docker,size=1G",
|
||||
"mp1": restored["mp1"], "mp8": restored["mp8"], "mp9": restored["mp9"],
|
||||
}
|
||||
_, delta = mountParity(archive, small)
|
||||
if len(delta) != 1 || !strings.Contains(delta[0], "mp0 size 1G < archive 200G") {
|
||||
t.Fatalf("undersized mpN must be a named delta, got %v", delta)
|
||||
}
|
||||
|
||||
// an extra restored mpN the archive never had is a delta too
|
||||
extra := map[string]string{
|
||||
"mp0": restored["mp0"], "mp1": restored["mp1"], "mp8": restored["mp8"], "mp9": restored["mp9"],
|
||||
"mp3": "local-lvm:vm-990000-disk-9,mp=/stray,size=1G",
|
||||
}
|
||||
_, delta = mountParity(archive, extra)
|
||||
if len(delta) != 1 || !strings.Contains(delta[0], "mp3 present on the restored guest but not in the archive") {
|
||||
t.Fatalf("extra mpN must be a named delta, got %v", delta)
|
||||
}
|
||||
|
||||
// no mpN anywhere = trivially parity (the golden-shaped archive)
|
||||
inv, delta = mountParity(map[string]string{"rootfs": "l:d,size=8G"}, map[string]string{})
|
||||
if len(delta) != 0 || len(inv) != 0 {
|
||||
t.Fatalf("mpN-less archive must be trivial parity, got inv=%v delta=%v", inv, delta)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -529,3 +530,140 @@ func indexOf(s, sub string) int {
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// ── GL-5b: full-fidelity restore-test (archive-derived params + mount parity) ────────────────────
|
||||
|
||||
// gl5bArchiveCfg is a 9201-shaped archive config: rootfs + two storage mpN + the two structural binds.
|
||||
const gl5bArchiveCfg = `hostname: demo
|
||||
rootfs: local-lvm:vm-9201-disk-0,size=32G
|
||||
mp0: local-lvm:vm-9201-disk-1,mp=/var/lib/docker,backup=1,size=200G
|
||||
mp1: local-lvm:vm-9201-disk-2,mp=/mnt/sys_drive,backup=1,size=50G
|
||||
mp8: /mnt/felhom-drives,mp=/mnt/felhom-drives
|
||||
mp9: /var/lib/felhom-agent/guests/9201/bootstrap,mp=/etc/felhom-bootstrap,ro=1
|
||||
`
|
||||
|
||||
// gl5bRestoredCfg builds the scratch guest's restored config as PVE would report it after a
|
||||
// FULL-fidelity restore (storage mpN at archived path+size, binds as 1G throwaways) + a net0 so
|
||||
// the link-down step runs.
|
||||
func gl5bRestoredCfg() proxmox.GuestConfig {
|
||||
c := scratchCfg()
|
||||
c.Extra["mp0"] = json.RawMessage(`"local-lvm:vm-990000-disk-1,mp=/var/lib/docker,backup=1,size=200G"`)
|
||||
c.Extra["mp1"] = json.RawMessage(`"local-lvm:vm-990000-disk-2,mp=/mnt/sys_drive,backup=1,size=50G"`)
|
||||
c.Extra["mp8"] = json.RawMessage(`"local-lvm:vm-990000-disk-3,mp=/mnt/felhom-drives,backup=0,size=1G"`)
|
||||
c.Extra["mp9"] = json.RawMessage(`"local-lvm:vm-990000-disk-4,mp=/etc/felhom-bootstrap,backup=0,size=1G"`)
|
||||
return c
|
||||
}
|
||||
|
||||
// GL-5b Scenario A: the restore-test passes the FULL drRestoreOverrides param set (derived from
|
||||
// the ARCHIVE config, not any live guest) and the parity assert verifies the restored mpN set.
|
||||
func TestRunRestoreTest_FullFidelityParams(t *testing.T) {
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: gl5bRestoredCfg()}, extractCfg: gl5bArchiveCfg}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "local:backup/vzdump-lxc-9201-x.tar.zst", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
||||
})
|
||||
if res.Err != nil || !res.Pass {
|
||||
t.Fatalf("expected pass, got %+v", res)
|
||||
}
|
||||
if len(api.extracts) != 1 || api.extracts[0] != "local:backup/vzdump-lxc-9201-x.tar.zst" {
|
||||
t.Fatalf("params must derive from the ARCHIVE's extracted config: %+v", api.extracts)
|
||||
}
|
||||
ov := api.restores[0].MountOverrides
|
||||
want := map[string]string{
|
||||
"rootfs": "local-lvm:32",
|
||||
"mp0": "local-lvm:200,mp=/var/lib/docker,backup=1",
|
||||
"mp1": "local-lvm:50,mp=/mnt/sys_drive,backup=1",
|
||||
"mp8": "local-lvm:1,mp=/mnt/felhom-drives,backup=0",
|
||||
"mp9": "local-lvm:1,mp=/etc/felhom-bootstrap,backup=0",
|
||||
}
|
||||
if len(ov) != len(want) {
|
||||
t.Fatalf("MountOverrides = %+v, want %+v", ov, want)
|
||||
}
|
||||
for k, v := range want {
|
||||
if ov[k] != v {
|
||||
t.Errorf("override[%s] = %q, want %q", k, ov[k], v)
|
||||
}
|
||||
}
|
||||
if res.MountParity != "ok" {
|
||||
t.Errorf("MountParity = %q, want ok", res.MountParity)
|
||||
}
|
||||
if len(res.MountInventory) != 4 {
|
||||
t.Errorf("MountInventory must carry the 4 verified mpN, got %v", res.MountInventory)
|
||||
}
|
||||
// scratch still torn down (full-fidelity changes verification, not lifecycle)
|
||||
if len(api.destroys) != 1 || api.destroys[0] != 990000 {
|
||||
t.Errorf("scratch must be torn down: %+v", api.destroys)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5b Scenario B (the non-hollow core): a restored guest MISSING a storage mpN — exactly PVE's
|
||||
// drop-unlisted-mountpoints shape — boots green but must FAIL on parity, naming the mpN.
|
||||
// COMPANION RED-PROOF: with the 2b parity assert removed, this run passes silently (mutation
|
||||
// run→fail→revert recorded in the REPORT).
|
||||
func TestRunRestoreTest_ParityCatchesDroppedMount(t *testing.T) {
|
||||
c := gl5bRestoredCfg()
|
||||
delete(c.Extra, "mp0") // constraint-(b): the docker-data volume silently dropped
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: c}, extractCfg: gl5bArchiveCfg}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "local:backup/vzdump-lxc-9201-x.tar.zst", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
||||
})
|
||||
if res.Pass || res.Err == nil {
|
||||
t.Fatalf("a dropped mpN must FAIL the test, got %+v", res)
|
||||
}
|
||||
if !strings.Contains(res.Err.Error(), "mount parity FAILED") || !strings.Contains(res.Err.Error(), "mp0 MISSING") {
|
||||
t.Fatalf("the verdict must name the missing mpN, got: %v", res.Err)
|
||||
}
|
||||
if res.MountParity != "mismatch" {
|
||||
t.Errorf("MountParity = %q, want mismatch", res.MountParity)
|
||||
}
|
||||
// the guest never boots (parity fails pre-start) and the scratch is still torn down
|
||||
if len(api.starts) != 0 {
|
||||
t.Errorf("a parity-failed scratch must not be started: %+v", api.starts)
|
||||
}
|
||||
if len(api.destroys) != 1 {
|
||||
t.Errorf("teardown must still fire (launch-proven): %+v", api.destroys)
|
||||
}
|
||||
}
|
||||
|
||||
// GL-5b Scenario C: refusals propagate — an unreadable archive config or an unknown-topology
|
||||
// archive refuses UP FRONT (no restore, no scratch, nothing to tear down).
|
||||
func TestRunRestoreTest_RefusalsPropagate(t *testing.T) {
|
||||
// unreadable archive config
|
||||
apiErr := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}}
|
||||
apiErr.extractErr = errors.New("proxmox: GET extractconfig -> HTTP 500: volume not found")
|
||||
e1, _, q1 := newEngine(t, apiErr, EmptyProvider{})
|
||||
defer q1.Close()
|
||||
res := e1.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "local:backup/gone.tar.zst", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
||||
})
|
||||
if res.Err == nil || !strings.Contains(res.Err.Error(), "extract archive config") {
|
||||
t.Fatalf("unreadable archive config must refuse naming the step, got %+v", res)
|
||||
}
|
||||
if len(apiErr.restores) != 0 || len(apiErr.destroys) != 0 {
|
||||
t.Fatalf("the refusal must fire BEFORE any restore/teardown: %+v %+v", apiErr.restores, apiErr.destroys)
|
||||
}
|
||||
|
||||
// unknown bind topology
|
||||
apiBind := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()},
|
||||
extractCfg: "rootfs: l:d,size=8G\nmp3: /srv/other,mp=/data\n"}
|
||||
e2, _, q2 := newEngine(t, apiBind, EmptyProvider{})
|
||||
defer q2.Close()
|
||||
res = e2.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
||||
})
|
||||
if res.Err == nil || !strings.Contains(res.Err.Error(), "unknown bind mountpoint") {
|
||||
t.Fatalf("unknown topology must refuse via drRestoreOverrides' error, got %+v", res)
|
||||
}
|
||||
if len(apiBind.restores) != 0 {
|
||||
t.Fatalf("never restore a partial guest to verify it: %+v", apiBind.restores)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user