package reconcile import ( "context" "fmt" "math" "sort" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // The self-restore-test (doc 03 §8) — the piece that closes "a backup you haven't restored // isn't a backup". It is a JOURNALED reconcile job so it inherits the slice-4 journal, // per-guest serialization, and crash-safe recovery: a mid-test crash can't leak a scratch // guest (engine.Recover tears it down). Every step here is BENIGN — restore-to-new // (ClassCreate), a benign net-link-down SetConfig, and a scratch teardown that is benign by // agent-tagged-scratch provenance (no new destructive class, no new crypto). // scratchKind is the journal Kind for a restore-test scratch-guest-owning entry. Recover // keys off JournalEntry.Scratch (not this string), but the Kind aids audit/debug. const scratchKind = "scratch_restore_test" // DefaultBootTimeout bounds how long the restore-test waits for the scratch guest to reach // running before declaring the verify failed. const DefaultBootTimeout = 2 * time.Minute // RestoreTestSpec parameterizes one restore-test. type RestoreTestSpec struct { Archive string // source archive volid to restore (resolved by the caller) SourceTier string // "local" this slice (pbs = Phase B) — for the report RestoreStorage string // target storage for the restored rootfs (e.g. "local-lvm") ScratchMin int // inclusive scratch VMID band (must be > 0) ScratchMax int // inclusive BootTimeout time.Duration // 0 → DefaultBootTimeout // RestoreTaskTimeout bounds the wait on the restore (vzrestore) task. 0 → WaitOptions' 10m // default (fine for a LOCAL restore). A WAN/pbs restore of a large guest runs long, so the // caller sets this generously for the pbs tier — else the wait expires mid-restore, teardown // fires against a still-restoring (not-yet-pool-associated) guest, and the scratch leaks. RestoreTaskTimeout time.Duration } // RestoreTestResult is the reconcile-local outcome (the backup package maps it to the // hub.RestoreTest wire record — reconcile must not import hub for this). type RestoreTestResult struct { Archive string SourceTier string ScratchVMID int Pass bool Verified string // "boot+running" this slice Skipped bool // no free scratch VMID in band → test not run Err error StartedAt time.Time Duration time.Duration // StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the // systemd-nesting advisory). Populated only when the start exited "WARNINGS: N"; // always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning). StartWarnings []string // WarningsRecognized is true iff every StartWarnings line matches the benign anchor. // It affects VISIBILITY ONLY (log level / operator attention), never the verdict — so a // wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and // never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A). WarningsRecognized bool // MountParity is "ok" when the restored scratch's mpN set matches the archive's (GL-5b: the // assert that keeps PVE's drop-unlisted-mountpoints rule from ever regressing into a green // light); "mismatch" fails the test with Err naming the delta. "" on runs that never reached // the assert (restore failed earlier). MountParity string // MountInventory lists the parity-verified mountpoints ("mpN= (G)" / throwaway // stand-ins) — the record's proof of WHAT was verified, not just that something booted. MountInventory []string } // benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start // advisory ("Systemd detected. You may need to enable nesting."). It carries no systemd // version number, so — unlike an exact-string allowlist on "Systemd 257…" — it cannot rot back // into the false-fail bug as guests move to systemd 258+. Matched case-insensitively. const benignWarningAnchor = "enable nesting" // extractWarningLines pulls the warning lines out of a task log tail. PVE prefixes task // warnings with "WARN" (e.g. "WARN: Systemd 257 detected…"); we keep those, trimmed. func extractWarningLines(logTail []string) []string { var out []string for _, l := range logTail { if t := strings.TrimSpace(l); strings.HasPrefix(t, "WARN") { out = append(out, t) } } return out } // warningsRecognized reports whether EVERY warning line is the benign anchor. Empty ⇒ true // (no warnings to worry about). One unrecognized line ⇒ false (operator should look). func warningsRecognized(warnings []string) bool { for _, w := range warnings { if !strings.Contains(strings.ToLower(w), benignWarningAnchor) { return false } } return true } // IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch // guest: ClassGuestDestroy made benign by AgentTaggedScratch provenance (classify.go). The // gate authorizes it unsigned but is genuinely in-path (wrong provenance → pending_signature). func IntentForScratchDestroy(hostID string, vmid int) Intent { return Intent{ Class: ClassGuestDestroy, HostID: hostID, GuestID: strconv.Itoa(vmid), VMID: vmid, Provenance: Provenance{AgentTaggedScratch: true}, // agent-internal, never hub-sourced Source: SourceOneShotJob, } } // RunRestoreTest runs one restore-test on the per-guest queue lane of a fresh scratch VMID. // It journals a Scratch-owned entry BEFORE any mutation, so a crash anywhere after this // point is recoverable (Recover destroys a launch-proven scratch guest via its journaled // UPID). Teardown runs on every launch-proven path (defer), including a failed verify — but // NEVER when the restore failed before creating anything (proof-of-launch, campaign F1b). A // band vmid PVE reports "already exists" is advanced past, not failed (F2). The returned Err // is the TEST verdict's error (restore or boot failure), independent of teardown success. func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) RestoreTestResult { now := time.Now().UTC() res := RestoreTestResult{Archive: spec.Archive, SourceTier: spec.SourceTier, StartedAt: now} if spec.Archive == "" || spec.RestoreStorage == "" { res.Err = fmt.Errorf("reconcile: restore-test needs an archive and a restore storage") return res } if spec.ScratchMin <= 0 || spec.ScratchMax < spec.ScratchMin { res.Err = fmt.Errorf("reconcile: invalid scratch VMID band [%d,%d]", spec.ScratchMin, spec.ScratchMax) return res } lxc, err := e.api.ListLXC(ctx) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test list guests: %w", err) return res } // Band-advance loop (campaign pool-effects F2): the band scan below is POOL-BLIND (the // scoped token's ListLXC can't see non-pool guests), so a band vmid can look free while a // squatter sits on it. PVE tells us at restore time ("already exists"); we then advance to // the next band vmid instead of failing — one squatter must not permanently break the // restore-test or raise a false "backup unrestorable" alert. Bounded by the band width. occupied := make(map[int]bool) for { vmid, ok := pickScratchVMID(lxc, spec.ScratchMin, spec.ScratchMax, occupied) if !ok { // Band exhausted (in-use and/or invisible squatters) → skip, never panic, never // pick out-of-band, never FAIL. Recover will reap any genuinely leaked ones. e.logger.Warn("restore-test skipped: no free scratch VMID in band", "min", spec.ScratchMin, "max", spec.ScratchMax, "occupied_invisible", len(occupied)) res.Skipped = true res.ScratchVMID = 0 res.Err = nil return res } res.ScratchVMID = vmid // Serialize on the scratch VMID's lane (inherits §10), and capture the result. var vmidOccupied bool ch := e.queue.Submit(vmid, func() error { vmidOccupied = e.runScratchTest(ctx, vmid, spec, &res) return res.Err }) <-ch if !vmidOccupied { break } e.logger.Warn("restore-test: band VMID occupied by a guest invisible to the token; advancing", "vmid", vmid) occupied[vmid] = true } res.Duration = time.Since(now) return res } // runScratchTest is the journaled body (runs on vmid's queue lane). The occupied return is true // ONLY when PVE synchronously refused the restore because the vmid already holds a guest (one // the pool-blind band scan couldn't see) — the caller then advances to the next band vmid (F2). func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) (occupied bool) { base := JournalEntry{OpID: e.scratchOpID(vmid), VMID: vmid, Kind: scratchKind, Scratch: true} // OWN the scratch guest's cleanup BEFORE any mutation. From here, a crash is recoverable. e.append(withState(base, OpStarted)) // Teardown runs on every exit AFTER the restore launched (even on a failed verify), using a // cancel-immune context so a daemon shutdown mid-test still tears down; if teardown fails, // the entry stays in-flight and Recover reaps the guest on the next start (via the journaled // UPID). `launched` is the proof-of-launch gate (campaign pool-effects F1b): a restore that // failed synchronously (no UPID) created NOTHING, so teardown must NEVER destroy the vmid — // an invisible pre-existing guest may sit there. The entry is then closed terminal-failed // (nothing exists to recover). launched := false defer func() { if launched { e.teardownScratch(ctx, base) return } e.append(withState(base, OpFailed)) }() // 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error // detection only — it does NOT make the Scratch entry terminal (teardown does). // The restore params derive from the ARCHIVE's own embedded config — the object under // test — exactly like DR bring-up (GL-5b closes GL-5 finding #2's mirror image): PVE's // explicit-params restore REQUIRES an explicit rootfs AND silently DROPS unlisted // mountpoints, so the pre-v0.76.0 live-source-config bind-override path boot-verified // scratch guests WITHOUT their storage mpN — weaker verification than it claimed. The FULL // layout now rides: rootfs explicit, every storage mpN passed through (its content is // genuinely EXTRACTED — full fidelity; the added runtime IS the verification), the two // structural binds → throwaway stand-ins. An unreadable archive config or an unknown // topology REFUSES up front — never restore a partial guest to "verify" it. rawCfg, err := e.api.ExtractArchiveConfig(ctx, spec.Archive) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test extract archive config: %w", err) return false } mountOverrides, err := drRestoreOverrides(rawCfg, spec.RestoreStorage) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test: %w", err) return false } e.logger.Info("restore-test: full-fidelity restore params derived from the archive config", "scratch", vmid, "params", len(mountOverrides)) upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{ // Pool=DefaultPool so the scratch guest is created INTO the felhom pool — else a pool-scoped // token 403s on the scratch guest's config/start/destroy (SPIKE residual #2). VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage, MountOverrides: mountOverrides, Pool: DefaultPool, // onboot=0 from the instant the guest exists. Step 2 below link-downs every NIC before the // guest is ever started, so the NORMAL path cannot conflict with the live source. This // covers the ABNORMAL path: a restore that fails before step 2 (e.g. the wait expiring) can // leave a scratch carrying the source's `onboot: 1` plus its MAC/static island IP/hostname — // which a host reboot would then start alongside the original. Observed live 2026-07-26. ConfigOverrides: map[string]string{"onboot": "0"}, }) if err != nil { if pveAlreadyExists(err) { // The band vmid holds a guest the pool-blind scan couldn't see. Nothing was // created; NOT a test verdict — the caller advances to the next band vmid (F2). return true } res.Err = fmt.Errorf("reconcile: restore-test restore: %w", err) return false } // Proof-of-launch: the POST was accepted — from here teardown owns the guest. Accepted // residual: a crash before the next append leaks a scratch guest Recover won't destroy // (no journaled UPID) — cleanable, and preferable to destroying an innocent guest. launched = true e.append(withUPID(base, upid, OpTaskRunning)) if upid != "" { if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: spec.RestoreTaskTimeout}); err != nil { res.Err = fmt.Errorf("reconcile: restore-test restore task: %w", err) return } } // 2. Net link-down on every interface BEFORE boot — test-safety so the clone (which // keeps the source MAC/hostname; identity-reset is slice 7) can't conflict with a // running source on L2/IP. Benign SetConfig. cfg, err := e.api.GuestConfig(ctx, vmid) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test read scratch config: %w", err) return } // 2b. Mount-parity assert (GL-5b, the non-hollow core): the restored guest's mpN set must // match the archive's — storage mpN as real volumes at their archived path+size, the two // structural binds as their throwaway stand-ins. This makes GL-5's PVE rule (b) — // unlisted mountpoints silently dropped — impossible to regress into a green light: a // boot-only verify cannot see a missing data volume; this can. inventory, delta := mountParity(archiveCurrentConfig(rawCfg), cfg.MountPoints()) res.MountInventory = inventory if len(delta) > 0 { res.MountParity = "mismatch" res.Err = fmt.Errorf("reconcile: restore-test mount parity FAILED (restored scratch does not match the archive): %s", strings.Join(delta, "; ")) return } res.MountParity = "ok" for key, val := range cfg.Nets() { if _, err := e.api.SetConfig(ctx, vmid, map[string]string{key: withLinkDown(val)}); err != nil { res.Err = fmt.Errorf("reconcile: restore-test net link-down %s: %w", key, err) return } } // 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8). // The VERDICT is liveness (waitRunning), NEVER the start task's exitstatus. A start // that completes with warnings (e.g. the systemd-nesting advisory → exit "WARNINGS: N") // and then reaches running is a PASS — deciding pass/fail on an advisory exit code is // the crying-wolf bug this guards against. We pass AllowWarnings so WaitTask doesn't // hard-fail on it, then fetch + surface the warning text (visibility only). startUPID, err := e.api.Start(ctx, vmid) if err != nil { res.Err = fmt.Errorf("reconcile: restore-test start: %w", err) return } if startUPID != "" { st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true}) if err != nil { // A real (non-WARNINGS) start-task failure still fails the test. res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err) return } if strings.HasPrefix(st.ExitStatus, "WARNINGS") { // Surface the warning(s); do NOT fail. Liveness below is the verdict. tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50) if logErr != nil { e.logger.Warn("restore-test: could not read start-task log for warnings", "vmid", vmid, "err", logErr) } res.StartWarnings = extractWarningLines(tail) res.WarningsRecognized = warningsRecognized(res.StartWarnings) } } if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil { res.Err = err return } res.Pass = true res.Verified = "boot+running" return false } // throwawayVolumeOverride is the ONE source of the restore-override value format: a throwaway 1G // volume on restoreStorage at the given in-guest path, excluded from backup. Used by // drRestoreOverrides (bringup.go) for the structural binds — which both the DR bring-up and the // restore-test (GL-5b) derive their restore params from. (The old live-source-config // bindMountOverrides/archiveVMID path was deleted in v0.76.0 when the restore-test switched to // archive-derived params: it verified the wrong object AND tripped PVE's drop-unlisted-mountpoints // rule; keeping the dead lookalike reachable is how the next bug happens.) func throwawayVolumeOverride(restoreStorage, guestPath string) string { return fmt.Sprintf("%s:1,mp=%s,backup=0", restoreStorage, guestPath) } // mountParity compares the restored scratch guest's mpN set against the archive's current config // (GL-5b). Per archive mpN: a storage-backed one must exist restored at the same in-guest path // with at least the archived size (the pass-through recreates it at exactly that size); a // structural host bind must exist as its throwaway stand-in at the same path. Restored mpN slots // the archive doesn't carry are a delta too. Returns the verified inventory + the mismatch list // (empty delta = parity). Deterministic order (sorted slots). func mountParity(archiveCfg, restored map[string]string) (inventory, delta []string) { isMP := func(k string) bool { return len(k) > 2 && k[:2] == "mp" && k[2] >= '0' && k[2] <= '9' } slots := make([]string, 0, len(archiveCfg)) for k := range archiveCfg { if isMP(k) { slots = append(slots, k) } } sort.Strings(slots) for _, k := range slots { aval := archiveCfg[k] avol, arest, _ := strings.Cut(aval, ",") apath := mountPathOf(arest) if apath == "" && strings.HasPrefix(avol, "/") { apath = avol // a bind without an explicit mp= mounts at its host path } rval, ok := restored[k] if !ok { delta = append(delta, fmt.Sprintf("%s MISSING from the restored guest (archive: %s)", k, aval)) continue } _, rrest, _ := strings.Cut(rval, ",") if rpath := mountPathOf(rrest); rpath != apath { delta = append(delta, fmt.Sprintf("%s path %q != archive %q", k, rpath, apath)) continue } if strings.HasPrefix(avol, "/") { // structural bind → its throwaway stand-in (a real volume at the same path) inventory = append(inventory, fmt.Sprintf("%s=%s (throwaway for the archived bind)", k, apath)) continue } asz, rsz := rootfsSizeGB(aval), rootfsSizeGB(rval) if asz > 0 && rsz < asz { delta = append(delta, fmt.Sprintf("%s size %dG < archive %dG", k, rsz, asz)) continue } inventory = append(inventory, fmt.Sprintf("%s=%s (%dG)", k, apath, rsz)) } for k, v := range restored { if isMP(k) && archiveCfg[k] == "" { delta = append(delta, fmt.Sprintf("%s present on the restored guest but not in the archive: %s", k, v)) } } sort.Strings(delta) return inventory, delta } // mountPathOf returns the mp= field from an mpN value's trailing options ("" if absent). func mountPathOf(opts string) string { for _, kv := range strings.Split(opts, ",") { if v, ok := strings.CutPrefix(kv, "mp="); ok { return v } } return "" } // rootfsSizeGB parses the GB size from a volume spec's "size=" field (e.g. // "local-lvm:vm-9201-disk-0,size=8G"), rounding UP to whole GB — a restore needs the target volume // >= the archive's. Returns 0 when no size field is present (caller then skips the override). func rootfsSizeGB(spec string) int { for _, kv := range strings.Split(spec, ",") { if v, ok := strings.CutPrefix(kv, "size="); ok { return sizeToGB(v) } } return 0 } // sizeToGB converts a PVE size string ("8G", "512M", "1T", "8192K") to whole GB, rounding up (min 1 // when positive). Returns 0 on a malformed value. func sizeToGB(s string) int { if s == "" { return 0 } mult := 1.0 // default GB if no recognized unit suffix num := s switch s[len(s)-1] { case 'T', 't': mult, num = 1024, s[:len(s)-1] case 'G', 'g': mult, num = 1, s[:len(s)-1] case 'M', 'm': mult, num = 1.0/1024, s[:len(s)-1] case 'K', 'k': mult, num = 1.0/(1024*1024), s[:len(s)-1] } f, err := strconv.ParseFloat(num, 64) if err != nil || f <= 0 { return 0 } gb := int(math.Ceil(f * mult)) if gb < 1 { gb = 1 } return gb } // teardownScratch destroys the scratch guest (benign, gated) and records the entry terminal. // On any teardown failure it leaves the entry in-flight so Recover reaps the guest later. func (e *Engine) teardownScratch(ctx context.Context, base JournalEntry) { // Cancel-immune + bounded, so a shutdown mid-test still tears down. tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) defer cancel() dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, base.VMID), nil) if !dec.Allowed { e.logger.Error("restore-test: scratch teardown refused by gate (unexpected); left for Recover", "vmid", base.VMID, "reason", dec.Reason) return } upid, err := e.api.DestroyLXC(tctx, base.VMID) if err != nil { e.logger.Error("restore-test: scratch teardown failed; left for Recover", "vmid", base.VMID, "err", err) return } if upid != "" { if _, err := e.api.WaitTask(tctx, upid, proxmox.WaitOptions{}); err != nil { e.logger.Error("restore-test: scratch teardown task failed; left for Recover", "vmid", base.VMID, "err", err) return } } e.append(withState(base, OpSucceeded)) e.logger.Info("restore-test: scratch guest torn down", "vmid", base.VMID) } // waitRunning polls GuestStatus until the guest is running or the timeout elapses. The poll // interval is 2s in production, but shrinks for short timeouts so it stays responsive. func (e *Engine) waitRunning(ctx context.Context, vmid int, timeout time.Duration) error { interval := 2 * time.Second if timeout < 4*interval { if interval = timeout / 4; interval < 10*time.Millisecond { interval = 10 * time.Millisecond } } deadline := time.Now().Add(timeout) t := time.NewTicker(interval) defer t.Stop() for { g, err := e.api.GuestStatus(ctx, vmid) if err == nil && g.Status == "running" { return nil } if time.Now().After(deadline) { if err != nil { return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s (last err: %w)", vmid, timeout, err) } return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s", vmid, timeout) } select { case <-ctx.Done(): return ctx.Err() case <-t.C: } } } // pickScratchVMID returns the lowest free VMID in [min,max], excluding the standing 9999 // scratch, any in-use guest, and the caller's exclude set (band vmids PVE reported occupied by // guests the pool-blind list can't see — the F2 band-advance). ok=false when the band is fully // occupied (the test is then skipped, never run out-of-band). func pickScratchVMID(lxc []proxmox.Guest, min, max int, exclude map[int]bool) (int, bool) { used := make(map[int]bool, len(lxc)) for _, g := range lxc { used[g.VMID] = true } for id := min; id <= max; id++ { if id == 9999 || used[id] || exclude[id] { continue } return id, true } return 0, false } // withLinkDown sets link_down=1 on a Proxmox netN config string, REPLACING any existing // link_down token (never blind-concatenating, so a re-applied/pre-set value can't produce a // malformed netN). func withLinkDown(netN string) string { parts := strings.Split(netN, ",") out := parts[:0] for _, p := range parts { if p == "" || strings.HasPrefix(p, "link_down=") { continue } out = append(out, p) } out = append(out, "link_down=1") return strings.Join(out, ",") } func bootTimeout(spec RestoreTestSpec) time.Duration { if spec.BootTimeout > 0 { return spec.BootTimeout } return DefaultBootTimeout } func (e *Engine) scratchOpID(vmid int) string { return "scratch-restore-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq) } // withState / withUPID build journal records from a base entry, preserving its identity + // Scratch flag. func withState(base JournalEntry, state OpState) JournalEntry { base.State = state base.At = time.Now().UTC() return base } func withUPID(base JournalEntry, upid string, state OpState) JournalEntry { base.UPID = upid base.State = state base.At = time.Now().UTC() return base }