diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7e68f..0db17ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ intent actually gates future remounts. New `ExecutorChain` lets the signed-jobs `felhom-opsign` builds decommission params from `-durable-id`. No controller/customer UI — the operator path is hub jobs-queue → signed-jobs runner. +**Restore-test now boot-verifies slice-10 enrolled guests (bind-mount mountpoints).** A guest whose +data drive is a host BIND mount (slice-10 P2 `mp0`) could not be vzrestore'd by the privsep token +("restoring 'mpN' to bind mount is only possible for root") — so the restore-test failed for every +enrolled guest, regardless of backup tier (surfaced during the felhom-pbs live validation). The +restore-test now reads the SOURCE guest config (vmid parsed from the archive volid — PBS `ct//` +and vzdump `vzdump-lxc--` forms) and passes `RestoreLXCOptions.MountOverrides` that neutralize +each bind-mount `mpN` to a throwaway 1G volume on the restore storage (needs no root; the boot-verify +doesn't need the drive's data, and the host paths would otherwise collide). Storage-backed mountpoints +are restored normally; best-effort (an unreadable source config restores as-is). `proxmox.RestoreLXC` +gained `MountOverrides`. Verified live: restore-test from felhom-pbs of bind-mounted guest 9201 → +boot+running PASS. + ## v0.27.0 — slice 10 P3: self-heal watchdog reconcile + 4-state intent model (2026-06-12) The storage watchdog goes from detect-only → detect-and-reconcile: the agent autonomously re-mounts an diff --git a/internal/proxmox/mutate.go b/internal/proxmox/mutate.go index 08d7bf8..b874040 100644 --- a/internal/proxmox/mutate.go +++ b/internal/proxmox/mutate.go @@ -34,6 +34,13 @@ type RestoreLXCOptions struct { Archive string // source archive volid, e.g. "local:backup/vzdump-lxc-9001-...tar.zst" Storage string // target storage for the rootfs, e.g. "local-lvm" Force bool // overwrite an existing VMID (destructive — caller must have authority) + // MountOverrides overrides specific mountpoints at restore time (mpN -> full value, e.g. + // "local-lvm:1,mp=/data,backup=0"). A restore param takes precedence over the archive's own + // mpN. The restore-test uses this to neutralize a SOURCE host bind-mount mountpoint (slice-10 + // data drive) — vzrestore refuses to restore a bind mount under the privsep token ("restoring + // 'mpN' to bind mount is only possible for root"); replacing it with a throwaway volume needs + // no root and the boot-verify doesn't need the drive's data. + MountOverrides map[string]string } // RestoreLXC restores an LXC from a vzdump/PBS archive via POST /nodes/{node}/lxc @@ -52,6 +59,9 @@ func (c *Client) RestoreLXC(ctx context.Context, opts RestoreLXCOptions) (string if opts.Force { v.Set("force", "1") } + for k, val := range opts.MountOverrides { + v.Set(k, val) // e.g. mp0 -> "local-lvm:1,mp=/data,backup=0" (overrides the archive's mp0) + } return c.dataString(ctx, http.MethodPost, "/nodes/"+c.node+"/lxc", v) } diff --git a/internal/reconcile/restoretest.go b/internal/reconcile/restoretest.go index 86c0b21..a4d9203 100644 --- a/internal/reconcile/restoretest.go +++ b/internal/reconcile/restoretest.go @@ -3,6 +3,7 @@ package reconcile import ( "context" "fmt" + "regexp" "strconv" "strings" "time" @@ -159,8 +160,27 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS // 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error // detection only — it does NOT make the Scratch entry terminal (teardown does). + // A source guest with a host BIND-mount mountpoint (slice-10 data drive) can't be + // vzrestore'd by the privsep token ("restoring 'mpN' to bind mount is only possible for + // root"). The restore-test only needs the guest to BOOT — the data drives are irrelevant + // (their host paths would also collide). So neutralize each source bind-mount mpN to a + // throwaway volume on the restore storage (needs no root). Best-effort: if the source + // config can't be read, restore as-is (a bind-mount guest then fails as before, in the verdict). + var mountOverrides map[string]string + if srcVMID, ok := archiveVMID(spec.Archive); ok { + if srcCfg, cerr := e.api.GuestConfig(ctx, srcVMID); cerr == nil { + mountOverrides = bindMountOverrides(srcCfg.MountPoints(), spec.RestoreStorage) + if len(mountOverrides) > 0 { + e.logger.Info("restore-test: neutralizing source bind-mount mountpoints for scratch restore", + "source_vmid", srcVMID, "scratch", vmid, "count", len(mountOverrides)) + } + } else { + e.logger.Warn("restore-test: could not read source config for mp overrides (restoring as-is)", + "source_vmid", srcVMID, "err", cerr) + } + } upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{ - VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage, + VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage, MountOverrides: mountOverrides, }) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test restore: %w", err) @@ -226,6 +246,59 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS res.Verified = "boot+running" } +// archiveVMID extracts the source VMID from a backup archive volid. Handles PBS volids +// (":backup/ct//" and the /vm// form) and vzdump file volids +// (":backup/vzdump-(lxc|qemu)--..."). Returns false when no vmid is found. +func archiveVMID(volid string) (int, bool) { + for _, re := range []*regexp.Regexp{pbsArchiveRE, vzdumpArchiveRE} { + if m := re.FindStringSubmatch(volid); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + return n, true + } + } + } + return 0, false +} + +var ( + pbsArchiveRE = regexp.MustCompile(`/(?:ct|vm)/(\d+)/`) + vzdumpArchiveRE = regexp.MustCompile(`vzdump-(?:lxc|qemu)-(\d+)-`) +) + +// bindMountOverrides maps each BIND-mount mpN (the volume part is an absolute host PATH, not a +// "storage:volume") to a throwaway 1G volume override on restoreStorage, preserving the in-guest +// mount path. Storage-backed mpN are left to restore normally (returns nil when there are none). +// This is what lets the restore-test boot-verify a slice-10 enrolled guest whose data drive is a +// host bind mount the privsep token can't otherwise restore. +func bindMountOverrides(mps map[string]string, restoreStorage string) map[string]string { + out := map[string]string{} + for key, val := range mps { + volPart, rest, _ := strings.Cut(val, ",") + if !strings.HasPrefix(volPart, "/") { + continue // "storage:volume" → a real volume, not a host bind mount + } + mp := mountPathOf(rest) + if mp == "" { + mp = volPart // fall back to the host path if no explicit in-guest mp= + } + out[key] = fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, mp) + } + if len(out) == 0 { + return nil + } + return out +} + +// mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent). +func mountPathOf(opts string) string { + for _, kv := range strings.Split(opts, ",") { + if v, ok := strings.CutPrefix(kv, "mp="); ok { + return v + } + } + return "" +} + // teardownScratch destroys the scratch guest (benign, gated) and records the entry terminal. // On any teardown failure it leaves the entry in-flight so Recover reaps the guest later. func (e *Engine) teardownScratch(ctx context.Context, base JournalEntry) { diff --git a/internal/reconcile/restoretest_mounts_test.go b/internal/reconcile/restoretest_mounts_test.go new file mode 100644 index 0000000..8859b53 --- /dev/null +++ b/internal/reconcile/restoretest_mounts_test.go @@ -0,0 +1,48 @@ +package reconcile + +import "testing" + +func TestArchiveVMID(t *testing.T) { + cases := map[string]struct { + volid string + want int + ok bool + }{ + "pbs ct": {"felhom-pbs:backup/ct/9201/2026-06-12T18:29:58Z", 9201, true}, + "pbs vm": {"felhom-pbs:backup/vm/142/2026-06-12T18:29:58Z", 142, true}, + "vzdump lxc": {"local:backup/vzdump-lxc-9201-2026_06_12-18_29_58.tar.zst", 9201, true}, + "vzdump qemu": {"local:backup/vzdump-qemu-100-2026_06_12.vma.zst", 100, true}, + "none": {"local:iso/whatever.iso", 0, false}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + got, ok := archiveVMID(c.volid) + if ok != c.ok || got != c.want { + t.Errorf("archiveVMID(%q) = (%d,%v), want (%d,%v)", c.volid, got, ok, c.want, c.ok) + } + }) + } +} + +func TestBindMountOverrides(t *testing.T) { + mps := map[string]string{ + "mp0": "/mnt/felhom-usb/felhom-data,mp=/mnt/felhom-usb", // bind mount → override + "mp1": "local-lvm:8,mp=/data,backup=0", // real volume → left alone + "mp2": "/srv/extra", // bind mount, no explicit mp= → use host path + } + out := bindMountOverrides(mps, "local-lvm") + if _, ok := out["mp1"]; ok { + t.Error("mp1 is a storage volume and must NOT be overridden") + } + if got, want := out["mp0"], "local-lvm:1,mp=/mnt/felhom-usb,backup=0"; got != want { + t.Errorf("mp0 override = %q, want %q", got, want) + } + if got, want := out["mp2"], "local-lvm:1,mp=/srv/extra,backup=0"; got != want { + t.Errorf("mp2 override = %q, want %q", got, want) + } + + // No bind mounts → nil (restore proceeds unchanged). + if bindMountOverrides(map[string]string{"mp0": "local-lvm:8,mp=/data"}, "local-lvm") != nil { + t.Error("expected nil when there are no bind mounts") + } +}