felhom-agent/internal/reconcile/restoretest.go

package reconcile

import (
	"context"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// The self-restore-test (doc 03 §8) — the piece that closes "a backup you haven't restored
// isn't a backup". It is a JOURNALED reconcile job so it inherits the slice-4 journal,
// per-guest serialization, and crash-safe recovery: a mid-test crash can't leak a scratch
// guest (engine.Recover tears it down). Every step here is BENIGN — restore-to-new
// (ClassCreate), a benign net-link-down SetConfig, and a scratch teardown that is benign by
// agent-tagged-scratch provenance (no new destructive class, no new crypto).

// scratchKind is the journal Kind for a restore-test scratch-guest-owning entry. Recover
// keys off JournalEntry.Scratch (not this string), but the Kind aids audit/debug.
const scratchKind = "scratch_restore_test"

// DefaultBootTimeout bounds how long the restore-test waits for the scratch guest to reach
// running before declaring the verify failed.
const DefaultBootTimeout = 2 * time.Minute

// RestoreTestSpec parameterizes one restore-test.
type RestoreTestSpec struct {
	Archive        string        // source archive volid to restore (resolved by the caller)
	SourceTier     string        // "local" this slice (pbs = Phase B) — for the report
	RestoreStorage string        // target storage for the restored rootfs (e.g. "local-lvm")
	ScratchMin     int           // inclusive scratch VMID band (must be > 0)
	ScratchMax     int           // inclusive
	BootTimeout    time.Duration // 0 → DefaultBootTimeout
	// RestoreTaskTimeout bounds the wait on the restore (vzrestore) task. 0 → WaitOptions' 10m
	// default (fine for a LOCAL restore). A WAN/pbs restore of a large guest runs long, so the
	// caller sets this generously for the pbs tier — else the wait expires mid-restore, teardown
	// fires against a still-restoring (not-yet-pool-associated) guest, and the scratch leaks.
	RestoreTaskTimeout time.Duration
}

// RestoreTestResult is the reconcile-local outcome (the backup package maps it to the
// hub.RestoreTest wire record — reconcile must not import hub for this).
type RestoreTestResult struct {
	Archive     string
	SourceTier  string
	ScratchVMID int
	Pass        bool
	Verified    string // "boot+running" this slice
	Skipped     bool   // no free scratch VMID in band → test not run
	Err         error
	StartedAt   time.Time
	Duration    time.Duration
	// StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the
	// systemd-nesting advisory). Populated only when the start exited "WARNINGS: N";
	// always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning).
	StartWarnings []string
	// WarningsRecognized is true iff every StartWarnings line matches the benign anchor.
	// It affects VISIBILITY ONLY (log level / operator attention), never the verdict — so a
	// wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and
	// never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A).
	WarningsRecognized bool
	// MountParity is "ok" when the restored scratch's mpN set matches the archive's (GL-5b: the
	// assert that keeps PVE's drop-unlisted-mountpoints rule from ever regressing into a green
	// light); "mismatch" fails the test with Err naming the delta. "" on runs that never reached
	// the assert (restore failed earlier).
	MountParity string
	// MountInventory lists the parity-verified mountpoints ("mpN=<path> (<N>G)" / throwaway
	// stand-ins) — the record's proof of WHAT was verified, not just that something booted.
	MountInventory []string
}

// benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start
// advisory ("Systemd <N> detected. You may need to enable nesting."). It carries no systemd
// version number, so — unlike an exact-string allowlist on "Systemd 257…" — it cannot rot back
// into the false-fail bug as guests move to systemd 258+. Matched case-insensitively.
const benignWarningAnchor = "enable nesting"

// extractWarningLines pulls the warning lines out of a task log tail. PVE prefixes task
// warnings with "WARN" (e.g. "WARN: Systemd 257 detected…"); we keep those, trimmed.
func extractWarningLines(logTail []string) []string {
	var out []string
	for _, l := range logTail {
		if t := strings.TrimSpace(l); strings.HasPrefix(t, "WARN") {
			out = append(out, t)
		}
	}
	return out
}

// warningsRecognized reports whether EVERY warning line is the benign anchor. Empty ⇒ true
// (no warnings to worry about). One unrecognized line ⇒ false (operator should look).
func warningsRecognized(warnings []string) bool {
	for _, w := range warnings {
		if !strings.Contains(strings.ToLower(w), benignWarningAnchor) {
			return false
		}
	}
	return true
}

// IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch
// guest: ClassGuestDestroy made benign by AgentTaggedScratch provenance (classify.go). The
// gate authorizes it unsigned but is genuinely in-path (wrong provenance → pending_signature).
func IntentForScratchDestroy(hostID string, vmid int) Intent {
	return Intent{
		Class:      ClassGuestDestroy,
		HostID:     hostID,
		GuestID:    strconv.Itoa(vmid),
		VMID:       vmid,
		Provenance: Provenance{AgentTaggedScratch: true}, // agent-internal, never hub-sourced
		Source:     SourceOneShotJob,
	}
}

// RunRestoreTest runs one restore-test on the per-guest queue lane of a fresh scratch VMID.
// It journals a Scratch-owned entry BEFORE any mutation, so a crash anywhere after this
// point is recoverable (Recover destroys a launch-proven scratch guest via its journaled
// UPID). Teardown runs on every launch-proven path (defer), including a failed verify — but
// NEVER when the restore failed before creating anything (proof-of-launch, campaign F1b). A
// band vmid PVE reports "already exists" is advanced past, not failed (F2). The returned Err
// is the TEST verdict's error (restore or boot failure), independent of teardown success.
func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) RestoreTestResult {
	now := time.Now().UTC()
	res := RestoreTestResult{Archive: spec.Archive, SourceTier: spec.SourceTier, StartedAt: now}

	if spec.Archive == "" || spec.RestoreStorage == "" {
		res.Err = fmt.Errorf("reconcile: restore-test needs an archive and a restore storage")
		return res
	}
	if spec.ScratchMin <= 0 || spec.ScratchMax < spec.ScratchMin {
		res.Err = fmt.Errorf("reconcile: invalid scratch VMID band [%d,%d]", spec.ScratchMin, spec.ScratchMax)
		return res
	}

	lxc, err := e.api.ListLXC(ctx)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: restore-test list guests: %w", err)
		return res
	}
	// Band-advance loop (campaign pool-effects F2): the band scan below is POOL-BLIND (the
	// scoped token's ListLXC can't see non-pool guests), so a band vmid can look free while a
	// squatter sits on it. PVE tells us at restore time ("already exists"); we then advance to
	// the next band vmid instead of failing — one squatter must not permanently break the
	// restore-test or raise a false "backup unrestorable" alert. Bounded by the band width.
	occupied := make(map[int]bool)
	for {
		vmid, ok := pickScratchVMID(lxc, spec.ScratchMin, spec.ScratchMax, occupied)
		if !ok {
			// Band exhausted (in-use and/or invisible squatters) → skip, never panic, never
			// pick out-of-band, never FAIL. Recover will reap any genuinely leaked ones.
			e.logger.Warn("restore-test skipped: no free scratch VMID in band",
				"min", spec.ScratchMin, "max", spec.ScratchMax, "occupied_invisible", len(occupied))
			res.Skipped = true
			res.ScratchVMID = 0
			res.Err = nil
			return res
		}
		res.ScratchVMID = vmid

		// Serialize on the scratch VMID's lane (inherits §10), and capture the result.
		var vmidOccupied bool
		ch := e.queue.Submit(vmid, func() error {
			vmidOccupied = e.runScratchTest(ctx, vmid, spec, &res)
			return res.Err
		})
		<-ch
		if !vmidOccupied {
			break
		}
		e.logger.Warn("restore-test: band VMID occupied by a guest invisible to the token; advancing",
			"vmid", vmid)
		occupied[vmid] = true
	}
	res.Duration = time.Since(now)
	return res
}

// runScratchTest is the journaled body (runs on vmid's queue lane). The occupied return is true
// ONLY when PVE synchronously refused the restore because the vmid already holds a guest (one
// the pool-blind band scan couldn't see) — the caller then advances to the next band vmid (F2).
func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) (occupied bool) {
	base := JournalEntry{OpID: e.scratchOpID(vmid), VMID: vmid, Kind: scratchKind, Scratch: true}

	// OWN the scratch guest's cleanup BEFORE any mutation. From here, a crash is recoverable.
	e.append(withState(base, OpStarted))

	// Teardown runs on every exit AFTER the restore launched (even on a failed verify), using a
	// cancel-immune context so a daemon shutdown mid-test still tears down; if teardown fails,
	// the entry stays in-flight and Recover reaps the guest on the next start (via the journaled
	// UPID). `launched` is the proof-of-launch gate (campaign pool-effects F1b): a restore that
	// failed synchronously (no UPID) created NOTHING, so teardown must NEVER destroy the vmid —
	// an invisible pre-existing guest may sit there. The entry is then closed terminal-failed
	// (nothing exists to recover).
	launched := false
	defer func() {
		if launched {
			e.teardownScratch(ctx, base)
			return
		}
		e.append(withState(base, OpFailed))
	}()

	// 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error
	//    detection only — it does NOT make the Scratch entry terminal (teardown does).
	//    The restore params derive from the ARCHIVE's own embedded config — the object under
	//    test — exactly like DR bring-up (GL-5b closes GL-5 finding #2's mirror image): PVE's
	//    explicit-params restore REQUIRES an explicit rootfs AND silently DROPS unlisted
	//    mountpoints, so the pre-v0.76.0 live-source-config bind-override path boot-verified
	//    scratch guests WITHOUT their storage mpN — weaker verification than it claimed. The FULL
	//    layout now rides: rootfs explicit, every storage mpN passed through (its content is
	//    genuinely EXTRACTED — full fidelity; the added runtime IS the verification), the two
	//    structural binds → throwaway stand-ins. An unreadable archive config or an unknown
	//    topology REFUSES up front — never restore a partial guest to "verify" it.
	rawCfg, err := e.api.ExtractArchiveConfig(ctx, spec.Archive)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: restore-test extract archive config: %w", err)
		return false
	}
	mountOverrides, err := drRestoreOverrides(rawCfg, spec.RestoreStorage)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: restore-test: %w", err)
		return false
	}
	e.logger.Info("restore-test: full-fidelity restore params derived from the archive config",
		"scratch", vmid, "params", len(mountOverrides))
	upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
		// Pool=DefaultPool so the scratch guest is created INTO the felhom pool — else a pool-scoped
		// token 403s on the scratch guest's config/start/destroy (SPIKE residual #2).
		VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage, MountOverrides: mountOverrides, Pool: DefaultPool,
		// onboot=0 from the instant the guest exists. Step 2 below link-downs every NIC before the
		// guest is ever started, so the NORMAL path cannot conflict with the live source. This
		// covers the ABNORMAL path: a restore that fails before step 2 (e.g. the wait expiring) can
		// leave a scratch carrying the source's `onboot: 1` plus its MAC/static island IP/hostname —
		// which a host reboot would then start alongside the original. Observed live 2026-07-26.
		ConfigOverrides: map[string]string{"onboot": "0"},
	})
	if err != nil {
		if pveAlreadyExists(err) {
			// The band vmid holds a guest the pool-blind scan couldn't see. Nothing was
			// created; NOT a test verdict — the caller advances to the next band vmid (F2).
			return true
		}
		res.Err = fmt.Errorf("reconcile: restore-test restore: %w", err)
		return false
	}
	// Proof-of-launch: the POST was accepted — from here teardown owns the guest. Accepted
	// residual: a crash before the next append leaks a scratch guest Recover won't destroy
	// (no journaled UPID) — cleanable, and preferable to destroying an innocent guest.
	launched = true
	e.append(withUPID(base, upid, OpTaskRunning))
	if upid != "" {
		if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: spec.RestoreTaskTimeout}); err != nil {
			res.Err = fmt.Errorf("reconcile: restore-test restore task: %w", err)
			return
		}
	}

	// 2. Net link-down on every interface BEFORE boot — test-safety so the clone (which
	//    keeps the source MAC/hostname; identity-reset is slice 7) can't conflict with a
	//    running source on L2/IP. Benign SetConfig.
	cfg, err := e.api.GuestConfig(ctx, vmid)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: restore-test read scratch config: %w", err)
		return
	}

	// 2b. Mount-parity assert (GL-5b, the non-hollow core): the restored guest's mpN set must
	//     match the archive's — storage mpN as real volumes at their archived path+size, the two
	//     structural binds as their throwaway stand-ins. This makes GL-5's PVE rule (b) —
	//     unlisted mountpoints silently dropped — impossible to regress into a green light: a
	//     boot-only verify cannot see a missing data volume; this can.
	inventory, delta := mountParity(archiveCurrentConfig(rawCfg), cfg.MountPoints())
	res.MountInventory = inventory
	if len(delta) > 0 {
		res.MountParity = "mismatch"
		res.Err = fmt.Errorf("reconcile: restore-test mount parity FAILED (restored scratch does not match the archive): %s", strings.Join(delta, "; "))
		return
	}
	res.MountParity = "ok"

	for key, val := range cfg.Nets() {
		if _, err := e.api.SetConfig(ctx, vmid, map[string]string{key: withLinkDown(val)}); err != nil {
			res.Err = fmt.Errorf("reconcile: restore-test net link-down %s: %w", key, err)
			return
		}
	}

	// 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8).
	//    The VERDICT is liveness (waitRunning), NEVER the start task's exitstatus. A start
	//    that completes with warnings (e.g. the systemd-nesting advisory → exit "WARNINGS: N")
	//    and then reaches running is a PASS — deciding pass/fail on an advisory exit code is
	//    the crying-wolf bug this guards against. We pass AllowWarnings so WaitTask doesn't
	//    hard-fail on it, then fetch + surface the warning text (visibility only).
	startUPID, err := e.api.Start(ctx, vmid)
	if err != nil {
		res.Err = fmt.Errorf("reconcile: restore-test start: %w", err)
		return
	}
	if startUPID != "" {
		st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
		if err != nil {
			// A real (non-WARNINGS) start-task failure still fails the test.
			res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err)
			return
		}
		if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
			// Surface the warning(s); do NOT fail. Liveness below is the verdict.
			tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
			if logErr != nil {
				e.logger.Warn("restore-test: could not read start-task log for warnings",
					"vmid", vmid, "err", logErr)
			}
			res.StartWarnings = extractWarningLines(tail)
			res.WarningsRecognized = warningsRecognized(res.StartWarnings)
		}
	}
	if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil {
		res.Err = err
		return
	}
	res.Pass = true
	res.Verified = "boot+running"
	return false
}

// throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G
// volume on restoreStorage at the given in-guest path, excluded from backup. Used by
// drRestoreOverrides (bringup.go) for the structural binds — which both the DR bring-up and the
// restore-test (GL-5b) derive their restore params from. (The old live-source-config
// bindMountOverrides/archiveVMID path was deleted in v0.76.0 when the restore-test switched to
// archive-derived params: it verified the wrong object AND tripped PVE's drop-unlisted-mountpoints
// rule; keeping the dead lookalike reachable is how the next bug happens.)
func throwawayVolumeOverride(restoreStorage, guestPath string) string {
	return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath)
}

// mountParity compares the restored scratch guest's mpN set against the archive's current config
// (GL-5b). Per archive mpN: a storage-backed one must exist restored at the same in-guest path
// with at least the archived size (the pass-through recreates it at exactly that size); a
// structural host bind must exist as its throwaway stand-in at the same path. Restored mpN slots
// the archive doesn't carry are a delta too. Returns the verified inventory + the mismatch list
// (empty delta = parity). Deterministic order (sorted slots).
func mountParity(archiveCfg, restored map[string]string) (inventory, delta []string) {
	isMP := func(k string) bool {
		return len(k) > 2 && k[:2] == "mp" && k[2] >= '0' && k[2] <= '9'
	}
	slots := make([]string, 0, len(archiveCfg))
	for k := range archiveCfg {
		if isMP(k) {
			slots = append(slots, k)
		}
	}
	sort.Strings(slots)
	for _, k := range slots {
		aval := archiveCfg[k]
		avol, arest, _ := strings.Cut(aval, ",")
		apath := mountPathOf(arest)
		if apath == "" && strings.HasPrefix(avol, "/") {
			apath = avol // a bind without an explicit mp= mounts at its host path
		}
		rval, ok := restored[k]
		if !ok {
			delta = append(delta, fmt.Sprintf("%s MISSING from the restored guest (archive: %s)", k, aval))
			continue
		}
		_, rrest, _ := strings.Cut(rval, ",")
		if rpath := mountPathOf(rrest); rpath != apath {
			delta = append(delta, fmt.Sprintf("%s path %q != archive %q", k, rpath, apath))
			continue
		}
		if strings.HasPrefix(avol, "/") {
			// structural bind → its throwaway stand-in (a real volume at the same path)
			inventory = append(inventory, fmt.Sprintf("%s=%s (throwaway for the archived bind)", k, apath))
			continue
		}
		asz, rsz := rootfsSizeGB(aval), rootfsSizeGB(rval)
		if asz > 0 && rsz < asz {
			delta = append(delta, fmt.Sprintf("%s size %dG < archive %dG", k, rsz, asz))
			continue
		}
		inventory = append(inventory, fmt.Sprintf("%s=%s (%dG)", k, apath, rsz))
	}
	for k, v := range restored {
		if isMP(k) && archiveCfg[k] == "" {
			delta = append(delta, fmt.Sprintf("%s present on the restored guest but not in the archive: %s", k, v))
		}
	}
	sort.Strings(delta)
	return inventory, delta
}

// mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent).
func mountPathOf(opts string) string {
	for _, kv := range strings.Split(opts, ",") {
		if v, ok := strings.CutPrefix(kv, "mp="); ok {
			return v
		}
	}
	return ""
}

// rootfsSizeGB parses the GB size from a volume spec's "size=<N><unit>" field (e.g.
// "local-lvm:vm-9201-disk-0,size=8G"), rounding UP to whole GB — a restore needs the target volume
// >= the archive's. Returns 0 when no size field is present (caller then skips the override).
func rootfsSizeGB(spec string) int {
	for _, kv := range strings.Split(spec, ",") {
		if v, ok := strings.CutPrefix(kv, "size="); ok {
			return sizeToGB(v)
		}
	}
	return 0
}

// sizeToGB converts a PVE size string ("8G", "512M", "1T", "8192K") to whole GB, rounding up (min 1
// when positive). Returns 0 on a malformed value.
func sizeToGB(s string) int {
	if s == "" {
		return 0
	}
	mult := 1.0 // default GB if no recognized unit suffix
	num := s
	switch s[len(s)-1] {
	case 'T', 't':
		mult, num = 1024, s[:len(s)-1]
	case 'G', 'g':
		mult, num = 1, s[:len(s)-1]
	case 'M', 'm':
		mult, num = 1.0/1024, s[:len(s)-1]
	case 'K', 'k':
		mult, num = 1.0/(1024*1024), s[:len(s)-1]
	}
	f, err := strconv.ParseFloat(num, 64)
	if err != nil || f <= 0 {
		return 0
	}
	gb := int(math.Ceil(f * mult))
	if gb < 1 {
		gb = 1
	}
	return gb
}

// teardownScratch destroys the scratch guest (benign, gated) and records the entry terminal.
// On any teardown failure it leaves the entry in-flight so Recover reaps the guest later.
func (e *Engine) teardownScratch(ctx context.Context, base JournalEntry) {
	// Cancel-immune + bounded, so a shutdown mid-test still tears down.
	tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
	defer cancel()

	dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, base.VMID), nil)
	if !dec.Allowed {
		e.logger.Error("restore-test: scratch teardown refused by gate (unexpected); left for Recover",
			"vmid", base.VMID, "reason", dec.Reason)
		return
	}
	upid, err := e.api.DestroyLXC(tctx, base.VMID)
	if err != nil {
		e.logger.Error("restore-test: scratch teardown failed; left for Recover", "vmid", base.VMID, "err", err)
		return
	}
	if upid != "" {
		if _, err := e.api.WaitTask(tctx, upid, proxmox.WaitOptions{}); err != nil {
			e.logger.Error("restore-test: scratch teardown task failed; left for Recover", "vmid", base.VMID, "err", err)
			return
		}
	}
	e.append(withState(base, OpSucceeded))
	e.logger.Info("restore-test: scratch guest torn down", "vmid", base.VMID)
}

// waitRunning polls GuestStatus until the guest is running or the timeout elapses. The poll
// interval is 2s in production, but shrinks for short timeouts so it stays responsive.
func (e *Engine) waitRunning(ctx context.Context, vmid int, timeout time.Duration) error {
	interval := 2 * time.Second
	if timeout < 4*interval {
		if interval = timeout / 4; interval < 10*time.Millisecond {
			interval = 10 * time.Millisecond
		}
	}
	deadline := time.Now().Add(timeout)
	t := time.NewTicker(interval)
	defer t.Stop()
	for {
		g, err := e.api.GuestStatus(ctx, vmid)
		if err == nil && g.Status == "running" {
			return nil
		}
		if time.Now().After(deadline) {
			if err != nil {
				return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s (last err: %w)", vmid, timeout, err)
			}
			return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s", vmid, timeout)
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-t.C:
		}
	}
}

// pickScratchVMID returns the lowest free VMID in [min,max], excluding the standing 9999
// scratch, any in-use guest, and the caller's exclude set (band vmids PVE reported occupied by
// guests the pool-blind list can't see — the F2 band-advance). ok=false when the band is fully
// occupied (the test is then skipped, never run out-of-band).
func pickScratchVMID(lxc []proxmox.Guest, min, max int, exclude map[int]bool) (int, bool) {
	used := make(map[int]bool, len(lxc))
	for _, g := range lxc {
		used[g.VMID] = true
	}
	for id := min; id <= max; id++ {
		if id == 9999 || used[id] || exclude[id] {
			continue
		}
		return id, true
	}
	return 0, false
}

// withLinkDown sets link_down=1 on a Proxmox netN config string, REPLACING any existing
// link_down token (never blind-concatenating, so a re-applied/pre-set value can't produce a
// malformed netN).
func withLinkDown(netN string) string {
	parts := strings.Split(netN, ",")
	out := parts[:0]
	for _, p := range parts {
		if p == "" || strings.HasPrefix(p, "link_down=") {
			continue
		}
		out = append(out, p)
	}
	out = append(out, "link_down=1")
	return strings.Join(out, ",")
}

func bootTimeout(spec RestoreTestSpec) time.Duration {
	if spec.BootTimeout > 0 {
		return spec.BootTimeout
	}
	return DefaultBootTimeout
}

func (e *Engine) scratchOpID(vmid int) string {
	return "scratch-restore-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq)
}

// withState / withUPID build journal records from a base entry, preserving its identity +
// Scratch flag.
func withState(base JournalEntry, state OpState) JournalEntry {
	base.State = state
	base.At = time.Now().UTC()
	return base
}
func withUPID(base JournalEntry, upid string, state OpState) JournalEntry {
	base.UPID = upid
	base.State = state
	base.At = time.Now().UTC()
	return base
}