felhom-agent/cmd/felhom-agent/main.go

// Command felhom-agent is the host agent.
//
// With no --selftest flag it runs as the daemon: the host-report poll loop
// (slice 3) that periodically POSTs a read-only host-report to the hub (the
// heartbeat). --selftest=read|task exercise the proxmox layer; --selftest=hub does
// one collect+report against the hub and prints what it would send.
package main

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
	"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
	"gitea.dooplex.hu/admin/felhom-agent/internal/config"
	"gitea.dooplex.hu/admin/felhom-agent/internal/desired"
	"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
	"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
	"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
	"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
	"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)

// version is the agent version. Overridable at build time with
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
var version = "0.60.0"

// runGuestHook is the PVE pre-start hook body (`felhom-agent guest-hook <vmid> <phase>`). On the
// pre-start phase it creates placeholder dirs for any absent bind-mount source so the guest always boots
// (the C1 net). It ALWAYS returns cleanly (exit 0) — a hook must never block a guest start. Heal output
// is written to stderr (PVE captures hook output into the task log).
func runGuestHook(args []string) {
	if len(args) < 2 {
		return
	}
	vmid, phase := args[0], args[1]
	if phase != guesthook.PhasePreStart {
		return
	}
	created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
	if len(created) > 0 {
		fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
	}
}

func main() {
	// Pre-start self-heal hook entrypoint. PVE invokes the registered hookscript as
	// `<bin> guest-hook <vmid> <phase>`. Handled BEFORE flag parsing — it takes positional args, must be
	// fast, needs no config/daemon/network, and must NEVER exit nonzero (a hook that fails would block
	// the guest start). See internal/guesthook (the C1 net).
	if len(os.Args) >= 2 && os.Args[1] == "guest-hook" {
		runGuestHook(os.Args[2:])
		return
	}
	var (
		cfgPath       string
		selftest      selftestFlag
		vmid          int
		watch         time.Duration
		archive       string
		mode          string
		hostname      string
		keep          bool
		rootfsGrow    int
		dataVolGrow   int
		dataVolMount  string
		sysDataGrow   int
		sysDataMount  string
		cores         int
		memoryMB      int
		pbsStorage    string
		paperkey      bool
		offline       bool
		upload        bool
		custID        string
		custDomain    string
		custName      string
		custEmail     string
		hubPassword   string
		blobPath      string
		expectedFP    string
		keyDest       string
		idBundlePath  string
		directivePath string
		swapImage     string
		showVersion   bool
	)
	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-sysdata-grow/-cores/-memory; keeps the guest)")
	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
	flag.StringVar(&mode, "mode", "provision", "for --selftest=bring-up: `provision` (golden, fresh identity) | `dr` (customer backup, preserve continuity)")
	flag.StringVar(&hostname, "hostname", "", "for --selftest=bring-up provision: the hostname to set on the new guest")
	flag.BoolVar(&keep, "keep", false, "for --selftest=bring-up: KEEP the guest instead of tearing it down at the end")
	flag.IntVar(&rootfsGrow, "rootfs-grow", 0, "for --selftest=bring-up|provision: grow the OS rootfs by this many GiB after restore (0 = keep golden size)")
	flag.IntVar(&dataVolGrow, "datavol-grow", 0, "for --selftest=bring-up|provision: grow the golden's Docker-data volume (mp0) by this many GiB (0 = keep golden size)")
	flag.StringVar(&dataVolMount, "datavol-mount", "", "for --selftest=bring-up|provision: the mpN slot of the Docker-data volume to grow (default mp0)")
	flag.IntVar(&sysDataGrow, "sysdata-grow", 0, "for --selftest=bring-up|provision: grow the golden's SSD user-data volume (mp1, /mnt/sys_drive) by this many GiB (0 = keep golden size)")
	flag.StringVar(&sysDataMount, "sysdata-mount", "", "for --selftest=bring-up|provision: the mpN slot of the user-data volume to grow (default mp1)")
	flag.IntVar(&cores, "cores", 0, "for --selftest=bring-up|provision: cap the guest to N CPU cores (0 = keep golden default). Applied in the pre-start config PUT.")
	flag.IntVar(&memoryMB, "memory", 0, "for --selftest=bring-up|provision: cap the guest RAM to N MiB (0 = keep golden default). Applied pre-start.")
	flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
	flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
	flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
	flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
	flag.StringVar(&blobPath, "blob", "", "for --selftest=escrow-consume: path to the R-wrapped escrow blob file")
	flag.StringVar(&expectedFP, "fingerprint", "", "for --selftest=escrow-consume: the EXPECTED key fingerprint (the gate target)")
	flag.StringVar(&keyDest, "keydest", "", "for --selftest=escrow-consume: where to install the recovered key (0600)")
	flag.StringVar(&idBundlePath, "identity-bundle", "", "for --selftest=escrow-create: a 0600 JSON file {tunnel_token,pbs_token} to ALSO escrow under R (10D)")
	flag.StringVar(&directivePath, "directive", "", "for --selftest=escrow-create: a JSON file with the non-secret DR directive (pbs repo/ns, expected fingerprint, tunnel id)")
	flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id — the hub config-pull target, baked into the guest's bootstrap")
	flag.StringVar(&hubPassword, "hub-password", "", "for --selftest=provision: the customer's hub RETRIEVAL PASSPHRASE (SECRET) — baked into bootstrap.json so the controller pulls its config (and the customer-scoped hub key) from the hub. The customer must already exist in the hub.")
	flag.StringVar(&swapImage, "image", "", "for --selftest=controller-swap: the target controller image ref (gitea.dooplex.hu/admin/felhom-controller:<semver>) — must already be pulled in the guest")
	flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)")
	flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)")
	flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)")
	flag.BoolVar(&showVersion, "version", false, "print version and exit")
	flag.Parse()

	if showVersion {
		fmt.Println("felhom-agent", version)
		return
	}

	cfg, err := config.Load(cfgPath)
	if err != nil {
		// A missing default config file is fine if env provides the values; only a
		// present-but-unreadable/invalid file is fatal here.
		if !(os.IsNotExist(errors.Unwrap(err)) && cfgPath == flag.Lookup("config").DefValue) {
			fmt.Fprintln(os.Stderr, "config error:", err)
			os.Exit(2)
		}
		cfg = config.Default()
	}
	logger := applog.New(cfg.LogLevel)

	switch selftest.mode {
	case "":
		os.Exit(runDaemon(cfg, logger))
	case "read":
		os.Exit(runSelftestRead(context.Background(), cfg, logger))
	case "task":
		os.Exit(runSelftestTask(context.Background(), cfg, logger, vmid))
	case "hub":
		os.Exit(runSelftestHub(context.Background(), cfg, logger))
	case "storage":
		os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
	case "backup":
		os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
	case "restore-test":
		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
	case "pbs-verify":
		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
	case "lanresolver":
		os.Exit(runSelftestLANResolver(context.Background(), cfg, logger, vmid))
	case "bring-up":
		os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep,
			bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
				SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB}))
	case "provision":
		os.Exit(runSelftestProvision(context.Background(), cfg, logger, provisionArgs{
			archive: archive, vmid: vmid, hostname: hostname,
			customerID: custID, hubPassword: hubPassword,
			sizing: bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
				SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB},
		}))
	case "escrow-create":
		os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath))
	case "escrow-consume":
		os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest))
	case "identity-consume":
		os.Exit(runSelftestIdentityConsume(context.Background(), logger, blobPath, keyDest))
	case "controller-swap":
		os.Exit(runSelftestControllerSwap(context.Background(), cfg, logger, vmid, swapImage))
	}
}

// newProxmoxClient builds the read-path proxmox client from config.
func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) {
	return proxmox.NewClient(proxmox.Config{
		Endpoint: cfg.Proxmox.Endpoint,
		Node:     cfg.Proxmox.Node,
		Token:    cfg.Proxmox.Token,
		TLS: proxmox.TLSConfig{
			CAFile:             cfg.Proxmox.TLS.CAFile,
			Fingerprint:        cfg.Proxmox.TLS.Fingerprint,
			InsecureSkipVerify: cfg.Proxmox.TLS.InsecureSkipVerify,
		},
	})
}

// newHostOps builds the privileged storage surface (slice 5 Phase B) from config. It shells
// out via the same fenced Runner the proxmox layer uses (sudo -n, arg vectors, no shell);
// every argument is validated in internal/storage before any command is built. A
// missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a
// crash.
// Returns the concrete *storage.SudoHostOps (not the HostOps interface) so callers that need the
// methods outside that lean interface — the host-reboot mount re-assert and the Part-A1 network-mount
// surface — can reach them without a type assertion. It still satisfies storage.HostOps everywhere.
func newHostOps(cfg config.Config, logger *slog.Logger) *storage.SudoHostOps {
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	return storage.NewSudoHostOps(storage.SudoHostOpsConfig{
		Runner: runner,
		Bins: storage.Binaries{
			Systemctl: cfg.Privileged.Systemctl,
			Install:   cfg.Privileged.Install,
			Smartctl:  cfg.Privileged.Smartctl,
			Lvs:       cfg.Privileged.Lvs,
		},
		UnitDir:  cfg.Privileged.UnitDir,
		StageDir: cfg.Privileged.StageDir,
		Logger:   logger,
	})
}

// gateRemounter is the watchdog's benign re-mount response: it routes the re-mount through
// the reversibility gate (classified benign) and, if allowed, calls HostOps.EnsureMount. It
// lives here (not in internal/storage) so storage stays decoupled from reconcile — main is
// the one place that holds both.
type gateRemounter struct {
	gate   *reconcile.Gate
	ops    storage.HostOps
	hostID string
	logger *slog.Logger
}

// Remount authorizes (benign) and performs a by-UUID re-mount of a returned target.
func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
	dec := r.gate.Authorize(reconcile.IntentForStorageMount(r.hostID, t.Name), nil)
	if !dec.Allowed {
		r.logger.Warn("storage: re-mount refused by gate (unexpected for a benign mount)",
			"target", t.Name, "reason", dec.Reason)
		return
	}
	uuid := t.UUID
	if uuid == "" {
		uuid = strings.TrimPrefix(t.DurableID, "uuid:") // durable_id carries it for usb/local-dir
	}
	spec := storage.MountSpec{Name: t.Name, UUID: uuid, Where: t.MountPath}
	if err := r.ops.EnsureMount(ctx, spec); err != nil {
		r.logger.Error("storage: re-mount failed", "target", t.Name, "where", t.MountPath, "err", err)
		return
	}
	r.logger.Info("storage: re-mounted returned target", "target", t.Name, "where", t.MountPath)
}

// logCapabilities logs the privileged-capability self-check at startup: one INFO summary, plus an
// ERROR per degraded capability naming the gated feature (so a missing grant is loud at cutover,
// not days later). It never exits — serve-degraded.
func logCapabilities(statuses []capability.Status, logger *slog.Logger) {
	ok, total, degraded := capability.Summarize(statuses)
	logger.Info("capabilities self-check", "ok", ok, "total", total, "degraded", len(degraded))
	for _, d := range degraded {
		logger.Error("capability DEGRADED — privileged grant missing (feature impaired until fixed)",
			"capability", d.Name, "feature", d.Feature, "reason", d.Reason, "critical", d.Critical)
	}
}

// runDaemon is the default mode: collect a host-report and POST it to the hub on a
// loop. Requires both proxmox (to collect) and hub config.
func runDaemon(cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "daemon: proxmox not configured:", err)
		return 2
	}
	if err := cfg.Hub.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "daemon: hub not configured:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: proxmox client:", err)
		return 1
	}
	client, err := hub.NewClient(cfg.Hub, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: hub client:", err)
		return 1
	}
	hcfg := cfg.Hub.WithDefaults()
	// Storage observer (slice 5): builds the report's storage_targets from Proxmox +
	// non-privileged host reads, enriched with the privileged SMART/lvs reads via HostOps
	// (Phase B). Wired into the collector via the StorageObserver seam (so hub does not
	// import storage).
	hostReader := storage.NewProcHostReader()
	hostOps := newHostOps(cfg, logger)
	observer := storage.NewObserver(px, hostReader, hostOps, logger)
	// Backup + restore-test state store (slice 6): holds the latest backup-per-target +
	// latest restore-test result; the collector reads it via the BackupReporter /
	// RestoreTestReporter seams; the cadence scheduler writes it.
	backupStore := backup.NewStore()
	// PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes the shared
	// store; the collector reads it via the PBSReporter seam. DR-recipe completion (v0.39.0): the
	// collector now reads through a LiveSnapshotReporter that lists snapshots LIVE each collect
	// (cheap GET, last-known-good fallback) so the host-report's pbs coord is present whenever PBS is
	// reachable — even seconds after a restart, before the 6 h verify loop has run. The verify loop
	// keeps Recording into the SAME store (shared last-known-good); targets are resolved once and
	// shared by both. pbsTargets is hoisted so the reporter and the verify loop use one closure.
	pbsStore := pbs.NewSnapshotStore()
	pbsTargets := pbsTargetsFromPVE(cfg, px, logger)
	pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargets, pbsStore, pbs.DefaultLiveSnapshotTimeout, logger)
	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsReporter, cfg.Hub.HostID, version, logger)
	// Privileged-capability self-check (v0.44.0): probe the sudoers grants the non-root agent
	// depends on. The probe runs `sudo -n -l` LITERALLY (a policy LIST, never executing the
	// command), so it uses a DIRECT runner regardless of the agent's privileged mode. Probe once at
	// startup (loud on any denial) and attach the snapshot to every hub report; the hub owns the
	// ok→degraded alert. Serve-degraded — a missing grant never blocks startup.
	capProber := capability.Prober{Runner: &proxmox.ExecRunner{Mode: proxmox.RunnerDirect}}
	logCapabilities(capProber.Probe(context.Background()), logger)
	collector.SetCapabilityProber(capProber.Probe)
	loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
	interval := time.Duration(hcfg.PollSeconds) * time.Second

	// Desired-state provider (slice 10A): the hub-served target the reconcile engine converges
	// toward. Starts EMPTY (generation 0) — reconcile is a live no-op until the hub serves intent,
	// exactly like the slice-4 EmptyProvider. The desired.Syncer (wired below) fills it when the
	// control envelope's generation advances. Replaces EmptyProvider in the engine.
	desiredProvider := reconcile.NewCachingProvider()
	// The "Down" channel sync hook: on each heartbeat, fetch desired-state when the generation
	// advances. The loop calls it via the EnvelopeObserver seam (hub does not import desired).
	desiredSyncer := desired.NewSyncer(client, desiredProvider, logger)
	// The signed-jobs runner (slice 10B) is wired as a SECOND envelope observer below (after the
	// gate is built) — when the heartbeat flags pending signed ops, it fetches + verifies + executes.

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	logger.Info("felhom-agent daemon starting",
		"version", version, "host_id", cfg.Hub.HostID, "hub_url", cfg.Hub.URL,
		"interval_s", hcfg.PollSeconds) // hub key intentionally not logged

	// Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue
	// (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving
	// until slice 10), so reconcile is a live no-op: it reads state and computes an
	// empty action set each tick, mutating nothing. The daemon must run cleanly with
	// no desired state and no signers configured — so a journal-open failure is logged
	// and reconcile proceeds journal-less (it has nothing destructive to journal yet).
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err != nil {
			logger.Warn("daemon: cannot ensure journal dir; reconcile runs without a journal", "path", jp, "err", err)
		} else if j, err := reconcile.OpenJournal(jp); err != nil {
			logger.Warn("daemon: cannot open op journal; reconcile runs without a journal", "path", jp, "err", err)
		} else {
			journal = j
			defer journal.Close()
		}
	}
	// The reversibility gate (slice 4 Phase B) sits in front of every mutation. With
	// no signers pinned (the common slice-4 state) the verifier is nil: benign actions
	// pass, destructive intents are refused pending_signature — and reconcile only
	// produces benign actions, so nothing is gated away. A misconfigured signer key is
	// a security misconfig and is fatal.
	verifier, nonceStore, err := buildVerifier(cfg, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: authz verifier:", err)
		return 2
	}
	if nonceStore != nil {
		defer nonceStore.Close()
	}
	gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)

	// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
	// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
	// known mount-backed target's device returning unmounted, dispatches a benign re-mount
	// (routed through the gate as benign, then HostOps). With no removable/network storage
	// it finds nothing to flag. The re-mount dispatch is off the poll path (a goroutine).
	storageTrigger := make(chan struct{}, 1)
	loop.SetTrigger(storageTrigger)
	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
	// Drive intent store (slice 10 P3 self-heal): persisted, durable-id-keyed enroll/eject/decommission
	// state. Gates the watchdog's self-heal re-mount to ENROLLED drives, and the local API records
	// enroll/eject into it. Open failure → ungated legacy remount (logged) so the daemon still runs.
	var intentReader storage.IntentReader
	intentStateDir := "/var/lib/felhom-agent"
	if cfg.Authz.NonceStorePath != "" {
		intentStateDir = filepath.Dir(cfg.Authz.NonceStorePath) // share the agent's durable state dir
	}
	intentStore, ierr := storage.OpenIntentStore(filepath.Join(intentStateDir, "drive-intents.json"))
	if ierr != nil {
		logger.Warn("storage: intent store unavailable — self-heal runs UNGATED (legacy remount-any)", "err", ierr)
		intentStore = nil
	} else {
		intentReader = intentStore
	}
	// F9: per-guest enrolled-bind record, replayed by ReassertGuestBinds at startup to restore a
	// guest data-drive bind that a re-provision dropped (durable-id-keyed; absent/swapped drives skipped).
	guestBindStore, gbErr := localapi.OpenGuestBindStore(filepath.Join(intentStateDir, "guest-binds.json"))
	if gbErr != nil {
		logger.Warn("storage: guest-bind store unavailable — startup bind re-assert disabled (F9)", "err", gbErr)
		guestBindStore = nil
	}
	// F20-BUG3: persisted disk-format job — lets mkfs run detached from the request and survive an agent
	// restart (RecoverFormatJob re-runs an interrupted durable-id-bound format).
	formatJobStore, fjErr := localapi.OpenFormatJobStore(filepath.Join(intentStateDir, "format-job.json"))
	if fjErr != nil {
		logger.Warn("storage: format-job store unavailable — format restart-recovery disabled (F20-BUG3)", "err", fjErr)
		formatJobStore = nil
	}
	// Impl-2a: source the watchdog's known-DRIVE set from the intent registry + Felhom .mount units
	// (NOT PVE storages via Observe) — a registry-only drive (no PVE dir-storage) is now health-tracked.
	// Real PVE storages (local/local-lvm/pbs) stay Observe-sourced for /disks + reports (observer below).
	driveUnitDir := cfg.Privileged.UnitDir
	if driveUnitDir == "" {
		driveUnitDir = "/etc/systemd/system"
	}
	driveKnown := storage.NewRegistryKnownTargets(driveUnitDir, intentReader, logger)
	// One-time migration: record pre-Impl-2a drives (mounted, with a Felhom unit) as enrolled so the
	// registry-sourced Known() tracks them without their legacy PVE dir-storage. Idempotent.
	if mounts, merr := hostReader.Mounts(); merr == nil {
		storage.ReconcileExistingDrives(driveUnitDir, mounts, intentStore, logger)
	} else {
		logger.Warn("storage: existing-drive migration skipped — mount read failed", "err", merr)
	}
	watchdog := storage.NewWatchdog(storage.WatchdogOptions{
		Targets:   storage.NewCachingKnownTargets(driveKnown, cfg.Storage.KnownRefresh()),
		Liveness:  storage.NewHostLiveness(hostReader, 0),
		Remounter: remounter,
		Intent:    intentReader, // P3: only re-mount enrolled drives
		OnAbsent: func(durableID string) { // P3: clear `ejected` on physical absence (replug rule)
			if intentStore != nil {
				_ = intentStore.OnAbsent(durableID)
			}
		},
		Trigger: func() {
			select {
			case storageTrigger <- struct{}{}:
			default:
			}
		},
		Interval: cfg.Storage.WatchdogInterval(),
		Debounce: cfg.Storage.WatchdogDebounce(),
		Logger:   logger,
	})

	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API:      px,
		Queue:    queue,
		Journal:  journal,
		Provider: desiredProvider, // slice 10A: hub-served desired-state (empty until the hub serves intent)
		Gate:     gate,
		HostID:   cfg.Hub.HostID,
		Logger:   logger,
	})

	// Crash recovery (doc 03 §10): resolve any op that was in flight when the agent
	// last died BEFORE issuing new mutations. For slice 6 this is load-bearing — a
	// restore-test scratch guest leaked by a mid-test crash is torn down here.
	engine.Recover(ctx)

	// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
	// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
	// OR the scratch band / restore storage is misconfigured — the daemon still runs.
	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)

	// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
	// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
	// verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs
	// storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing.
	pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
		Targets: pbsTargets, // the same closure the live reporter uses (shared store, one resolver)
		Store:   pbsStore,
		Cadence: cfg.Backup.PBSVerifyCadence(),
		Logger:  logger,
	})

	// Local API server (slice 8A, doc 03 §6): the per-guest authorization gate the in-guest
	// controller calls over the bridge. Optional — runs only when local_api.enable + listen_addr
	// are configured; a token-store or cert failure disables it WITHOUT killing the daemon (the
	// host still reports/reconciles). The leaf is generated+persisted once so its pin is stable.
	localServers := 0
	var localTokens *localapi.TokenStore
	var intentRec localapi.IntentRecorder
	if intentStore != nil { // avoid a typed-nil interface (would defeat the nil check)
		intentRec = intentStore
	}

	// Signed-jobs runner (slice 10B + P3): the consumer of the hub's signed-jobs queue. On a
	// heartbeat that flags pending signed ops, it fetches each opaque blob, runs it through the gate
	// (the LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time →
	// durable nonce-burn) and, only on all-pass, hands the verified op to the executor chain:
	//   - storage_wipe → re-resolves the DURABLE device id + re-inspects (8C) before mkfs (closes
	//     the 8C data-bearing `pending_signature` gap);
	//   - decommission → records the PERMANENT decommission intent (keyed by the watchdog's storage
	//     durable-id), making the previously-unreachable IntentDecommissioned state reachable ONLY
	//     via a verified operator signature (never customer-confirmable; distinct from a safe eject).
	// With no signers pinned the gate refuses every job (pending_signature) and nothing executes —
	// correct. Wired as a second envelope observer alongside the desired-state syncer. The intent
	// store is opened above (line ~340), so this wiring lives here (after it) rather than earlier.
	wipeExec := signedjobs.NewWipeExecutor(hostOps, logger)
	var decommIntent signedjobs.IntentDecommissioner
	if intentStore != nil { // typed-nil guard (a nil *IntentStore in the interface would pass != nil)
		decommIntent = intentStore
	}
	decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
	jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec}, cfg.Hub.HostID, logger)
	loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))

	localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logger, &localTokens)
	if localTokens != nil {
		defer localTokens.Close()
	}

	// LAN split-horizon resolver: the agent manages a host-side dnsmasq answering *.<customer-domain>
	// with each guest's live LAN IP (so LAN clients reach the box directly — same hostname, same real
	// wildcard cert — instead of hairpinning through Cloudflare). Optional; runs only when
	// lan_resolver.enable is set and a host LAN IP is known (explicit or derived from local_api).
	lanServers := 0
	var lanLoop *lanresolver.Loop
	{
		lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
		if lr.Enabled() && lr.HostIP != "" {
			lrMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if lrMode == "" {
				lrMode = proxmox.RunnerSudo
			}
			lrRunner := &proxmox.ExecRunner{Mode: lrMode, SudoPath: cfg.Privileged.SudoPath}
			mgr := lanresolver.NewManager(lrRunner, lr.HostIP, lr.Upstreams, logger)
			lanLoop = lanresolver.NewLoop(mgr, lr.StateDir, time.Duration(lr.IntervalSeconds)*time.Second, logger)
			logger.Info("lanresolver: enabled", "host_ip", lr.HostIP, "upstreams", lr.Upstreams, "interval_s", lr.IntervalSeconds)
		} else if lr.Enabled() {
			logger.Warn("lanresolver: enabled but no host IP (set lan_resolver.host_ip or local_api.listen_addr) — disabled")
		}
	}

	// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
	// verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop
	// concurrently; any one returning ends the daemon (ctx cancel tears down the rest).
	errc := make(chan error, 7)
	go func() { errc <- engine.Run(ctx, interval) }()
	go func() { errc <- loop.Run(ctx) }()
	go func() { errc <- watchdog.Run(ctx) }()
	go func() { errc <- scheduler.Run(ctx) }()
	go func() { errc <- pbsLoop.Run(ctx) }()
	if localSrv != nil {
		localServers = 1
		// Host-reboot remount fix: BEFORE binding into the guest, re-assert enrolled drive MOUNTS on the
		// host. A `disabled` mount unit (left so by a prior detach) doesn't auto-mount at boot, and kernel
		// re-enumeration can move the device (/dev/sdb→sdc); ReassertEnrolledMounts re-resolves each by
		// filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op.
		// Type-asserted (the concrete op exposes it; the interface stays lean).
		mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts
		if mountReasserter != nil {
			mountReasserter.ReassertEnrolledMounts(ctx)
		}
		// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
		// bind that a re-provision dropped — before serving, so the drive is back in the guest config
		// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
		localSrv.ReassertGuestBinds(ctx)
		// Intermediary-mount GUEST-REBOOT self-heal: re-run the reconcile periodically. A guest reboot
		// (without an agent restart) leaves enrolled drives bound on the HOST but INVISIBLE in the fresh
		// guest namespace (a non-recursive parent bind doesn't carry pre-existing submounts); the periodic
		// reconcile detects the guest can't see the bind and re-fires propagation (force re-bind). Cheap
		// when nothing changed (guest-visibility checks only). 20s ≪ the controller gate's 30s tick.
		go func() {
			t := time.NewTicker(20 * time.Second)
			defer t.Stop()
			for {
				select {
				case <-ctx.Done():
					return
				case <-t.C:
					// Re-assert mounts first (handles a USB that enumerated late after a host reboot, or a
					// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
					if mountReasserter != nil {
						mountReasserter.ReassertEnrolledMounts(ctx)
					}
					localSrv.ReassertGuestBinds(ctx)
				}
			}
		}()
		// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
		localSrv.RecoverFormatJob(ctx)
		// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete
		// the dangling snapshot → start iff onboot). Runs BEFORE the backup loop starts, so a present
		// backup lock is stale by definition (guarded by a no-vzdump-running check; fail-safe otherwise).
		localSrv.RecoverStaleLockedGuests(ctx)
		go func() { errc <- localSrv.Run(ctx) }()
	}
	if lanLoop != nil {
		lanServers = 1
		go func() { errc <- lanLoop.Run(ctx) }()
	}

	err = <-errc
	stop()                                           // tear down the siblings on the first exit
	for i := 0; i < 4+localServers+lanServers; i++ { // wait for the other goroutines
		<-errc
	}
	if err != nil && err != context.Canceled {
		logger.Error("daemon: exited with error", "err", err)
		return 1
	}
	return 0
}

// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs
// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each
// (token id from the storage `username`, secret read from <PBSSecretDir>/<id>.pw). A storage
// whose secret/client can't be built is skipped with a warning (the loop still verifies the
// rest). The secret is read at runtime, never logged.
func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets {
	return func(ctx context.Context) ([]pbs.Target, error) {
		stores, err := px.ListStorage(ctx)
		if err != nil {
			return nil, err
		}
		var targets []pbs.Target
		for _, s := range stores {
			if s.Type != "pbs" {
				continue
			}
			secret, err := readTrimmed(cfg.Backup.PBSSecretPath(s.Storage))
			if err != nil {
				logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
				continue
			}
			c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret})
			if err != nil {
				logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
				continue
			}
			targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c})
		}
		return targets, nil
	}
}

// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
	stores, err := px.ListStorage(ctx)
	if err != nil {
		return "local"
	}
	for _, s := range stores {
		if s.Storage == storageID && s.Type == "pbs" {
			return "pbs"
		}
	}
	return "local"
}

// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
func readTrimmed(path string) (string, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	s := strings.TrimSpace(string(b))
	if s == "" {
		return "", fmt.Errorf("empty file %s", path)
	}
	return s, nil
}

// buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
// machinery still works on-demand via --selftest=restore-test.
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, logger *slog.Logger) *backup.Scheduler {
	cadence := cfg.Backup.RestoreTestCadence()
	if cadence > 0 {
		if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
			logger.Warn("daemon: restore-test cadence disabled (config invalid)", "err", err)
			cadence = 0
		}
	}
	min, max := cfg.Backup.ScratchBand()
	target := cfg.Backup.BackupTarget()
	runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
	return backup.NewScheduler(backup.SchedulerOptions{
		Runner: engine,
		Pick:   runner.PickRestoreCandidate,
		Store:  store,
		Spec: reconcile.RestoreTestSpec{
			RestoreStorage: cfg.Backup.RestoreStorage,
			ScratchMin:     min,
			ScratchMax:     max,
			SourceTier:     storageTier(context.Background(), px, target),
		},
		Cadence: cadence,
		Logger:  logger,
	})
}

// buildLocalAPIServer constructs the per-guest local-API server (slice 8A, doc 03 §6) when
// configured. It opens the durable hashed token store and ensures the persisted self-signed
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
// fixed. The opened token store is returned via outTokens so the caller can Close it.
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
	if !cfg.LocalAPI.Enabled() {
		return nil
	}
	if err := cfg.LocalAPI.Validate(); err != nil {
		logger.Warn("daemon: local-api disabled (config invalid)", "err", err)
		return nil
	}
	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
	if err != nil {
		logger.Warn("daemon: local-api disabled (token store)", "err", err)
		return nil
	}
	*outTokens = tokens
	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
	cert, fp, generated, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
	if err != nil {
		logger.Warn("daemon: local-api disabled (leaf cert)", "err", err)
		return nil
	}
	if generated {
		// B.1: a freshly-minted leaf invalidates every already-issued bootstrap pin. LOUD so an
		// accidental regeneration (e.g. a state-dir move during a reinstall) is caught immediately.
		logger.Warn("local-api leaf REGENERATED — any previously issued bootstrap pins are now INVALID; controllers will fail the pin check until re-bootstrapped (or restore the prior leaf)",
			"fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
	} else {
		logger.Info("local-api leaf LOADED", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
	}
	// v0.48.0: ride the served leaf fp on every host report so the hub can detect a re-key fleet-wide.
	collector.SetLeafFingerprint(fp)
	runner := backup.NewBackupRunner(px, cfg.Backup.BackupTarget(), "", "felhom local-api", cfg.Backup.PruneBackupsSpec(), logger)
	// Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown
	// (same fenced ExecRunner the host-storage + provision back-half use).
	gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if gaMode == "" {
		gaMode = proxmox.RunnerSudo
	}
	guestBinder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
	srv, err := localapi.NewServer(localapi.Options{
		ListenAddr:    cfg.LocalAPI.ListenAddr,
		Cert:          cert,
		Guests:        px,
		Backups:       runner,
		Store:         store,
		Storage:       observer,
		DriveTargets:  driveTargets, // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages)
		HostReader:    storage.NewProcHostReader(), // Impl-2b: durableIDForMount raw-mount fallback + role gate
		Tokens:        tokens,
		BackupCadence: cfg.Backup.BackupCadence(),
		// Disk management (slice 8C): the privileged host surface + the data-bearing wipe gate.
		Disks:          hostOps,
		DiskGate:       storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID},
		Guests2:        px,
		GuestAttach:    guestBinder, // slice 10 P2: bind enrolled data drives into the guest
		// Network storage (NAS) — Part A1: the privileged host network-mount surface (NFS/SMB automount).
		NetStorage:  hostOps,
		SmbCredsDir: cfg.Privileged.SmbCredsDir,
		ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap
		// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
		// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
		StaleLock: localapi.NewStaleLockController(px, &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}),
		Intent:         intent,      // slice 10 P3: record enroll/eject intent for self-heal
		GuestBinds:     guestBinds,  // F9: per-guest bind record for the startup re-assert
		FormatJobs:     formatJobs,  // F20-BUG3: detached-format job record + restart recovery

		// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
		// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
		HostMetrics: collector,
		HostID:      cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op
		Logger:      logger,
	})
	if err != nil {
		logger.Warn("daemon: local-api disabled (server build)", "err", err)
		return nil
	}
	return srv
}

// storageGateAdapter bridges the local-API disk-format path to the reversibility gate, TIERED by the
// agent's authoritative device-role verdict: a USER-DATA data-bearing wipe is customer-confirmable
// (durable-id-bound, no signature); a SYSTEM/BACKUP wipe stays operator-signature (unsigned →
// pending_signature; the signed completion runs via the signed-jobs runner).
type storageGateAdapter struct {
	gate   *reconcile.Gate
	hostID string
}

func (a storageGateAdapter) AuthorizeWipe(req localapi.WipeRequest) localapi.WipeDecision {
	dec := a.gate.AuthorizeStorageWipe(reconcile.StorageWipeAuthz{
		HostID:           a.hostID,
		Role:             req.Role,
		DeviceDurableID:  req.DeviceDurableID,
		Confirmed:        req.Confirmed,
		ConfirmDurableID: req.ConfirmDurableID,
	}, nil) // no operator signature on the inline path → system/backup is pending_signature
	tier := "destructive"
	if dec.Disposition == reconcile.CustomerConfirmable {
		tier = "customer_confirmable"
	}
	return localapi.WipeDecision{
		Allowed:           dec.Allowed,
		Tier:              tier,
		Reason:            string(dec.Reason),
		NeedsConfirmation: dec.Reason == reconcile.ReasonPendingConfirmation,
	}
}

// reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
// configured nonce store (both are durable agent state), falling back to the standard
// host state dir when the nonce store is unset.
func reconcileJournalPath(cfg config.Config) string {
	if p := cfg.Authz.NonceStorePath; p != "" {
		return filepath.Join(filepath.Dir(p), "journal.log")
	}
	return "/var/lib/felhom-agent/journal.log"
}

// buildVerifier constructs the operator-signed-op verifier from config. With no signers
// pinned it returns (nil, nil, nil) — the gate then refuses any destructive intent as
// pending_signature (and reconcile serves none). With signers it requires a durable
// nonce-store path (anti-replay must survive restarts) and parses every pinned key —
// a bad key or missing path is a fatal security misconfig.
func buildVerifier(cfg config.Config, logger *slog.Logger) (reconcile.OpVerifier, *authz.FileNonceStore, error) {
	if len(cfg.Authz.Signers) == 0 {
		logger.Info("daemon: no operator signers pinned; destructive ops will be refused pending_signature")
		return nil, nil, nil
	}
	if cfg.Authz.NonceStorePath == "" {
		return nil, nil, fmt.Errorf("authz.nonce_store_path is required when signers are configured")
	}
	signers := make([]authz.AllowedSigner, 0, len(cfg.Authz.Signers))
	for _, s := range cfg.Authz.Signers {
		as, err := authz.NewAllowedSigner(s.KeyID, authz.KeyRole(s.Role), s.PublicKey)
		if err != nil {
			return nil, nil, fmt.Errorf("pinned signer %q: %w", s.KeyID, err)
		}
		signers = append(signers, as)
	}
	store, err := authz.OpenFileNonceStore(cfg.Authz.NonceStorePath)
	if err != nil {
		return nil, nil, fmt.Errorf("nonce store: %w", err)
	}
	logger.Info("daemon: operator signers pinned", "count", len(signers))
	return authz.New(signers, store, cfg.Hub.HostID), store, nil
}

// runSelftestControllerSwap exercises the agentic controller-update swap primitive directly (Phase 1):
// it swaps guest -vmid's controller to -image, verifies it comes up healthy, and ROLLS BACK if not.
// The target image must already be pulled in the guest (the controller pre-pulls it in the real flow).
func runSelftestControllerSwap(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int, image string) int {
	if vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=controller-swap: -vmid is required")
		return 1
	}
	if !localapi.ValidControllerImage(image) {
		fmt.Fprintln(os.Stderr, "selftest=controller-swap: -image must be gitea.dooplex.hu/admin/felhom-controller:<semver>")
		return 1
	}
	gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if gaMode == "" {
		gaMode = proxmox.RunnerSudo
	}
	binder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
	swapper := localapi.NewControllerSwapper(binder, "/var/lib/felhom-agent", logger)

	cur, _ := swapper.CurrentImage(ctx, vmid)
	fmt.Printf("=== felhom-agent %s selftest=controller-swap (vmid=%d) ===\n", version, vmid)
	fmt.Printf("  current image: %s\n  target  image: %s\n", cur, image)

	st := swapper.Swap(ctx, vmid, image)
	fmt.Printf("  --- result ---\n  state=%s current=%s previous=%s\n", st.State, st.Current, st.Previous)
	if st.Error != "" {
		fmt.Printf("  error: %s\n", st.Error)
	}
	if st.State == "done" {
		fmt.Println("=== selftest=controller-swap OK (swapped + healthy) ===")
		return 0
	}
	fmt.Println("=== selftest=controller-swap FAILED (rolled back if previous existed) ===")
	return 1
}

// runSelftestHub validates hub config, does ONE collect + report, and prints the
// report it would send plus the envelope it got back.
func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if err := cfg.Hub.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: hub not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	client, err := hub.NewClient(cfg.Hub, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: hub client:", err)
		return 1
	}
	observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
	// Wire the LIVE PBS reporter here too (v0.39.0): selftest=hub is a separate one-shot process — no
	// verify loop runs — so a nil reporter previously yielded pbs_snapshots:[] and an absent dr_recipe
	// pbs coord. The live reporter lists snapshots directly (fresh store, last-known-good fallback) so
	// the selftest reflects exactly what a freshly-restarted daemon's first collect emits.
	pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargetsFromPVE(cfg, px, logger), pbs.NewSnapshotStore(), pbs.DefaultLiveSnapshotTimeout, logger)
	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, pbsReporter, cfg.Hub.HostID, version, logger)

	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=hub (host_id=%s url=%s) ===\n", version, cfg.Hub.HostID, cfg.Hub.URL)
	report, err := collector.Collect(ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] collect:", err)
		return 1
	}
	if b, e := json.MarshalIndent(report, "  ", "  "); e == nil {
		fmt.Println("  --- report it would send ---")
		fmt.Println("  " + string(b))
	}
	env, err := client.Report(ctx, report)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] report:", err)
		return 1
	}
	if b, e := json.MarshalIndent(env, "  ", "  "); e == nil {
		fmt.Println("  --- control envelope received ---")
		fmt.Println("  " + string(b))
	}
	fmt.Println("=== selftest=hub OK ===")
	return 0
}

// runSelftestStorage is the live storage harness (slice 5 Phase B, for the USB runbook).
// It needs Proxmox config only (NO hub) so it runs standalone on the Proxmox host:
//   - observe pass: print the full StorageTarget table incl. the privileged SMART summary
//     and thin-pool data+metadata fill.
//   - -watch D: run the watchdog verbose for D with the re-mount response LIVE, so the
//     operator can physically cycle a drive and watch detect → report → re-mount.
func runSelftestStorage(ctx context.Context, cfg config.Config, logger *slog.Logger, watch time.Duration) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	hostReader := storage.NewProcHostReader()
	hostOps := newHostOps(cfg, logger)
	observer := storage.NewObserver(px, hostReader, hostOps, logger)

	octx, cancel := context.WithTimeout(ctx, 60*time.Second)
	fmt.Printf("=== felhom-agent %s selftest=storage ===\n", version)
	targets, err := observer.Observe(octx)
	cancel()
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] observe:", err)
		return 1
	}
	fmt.Printf("  observed %d storage target(s):\n", len(targets))
	for _, t := range targets {
		fmt.Printf("   - %-12s type=%-9s state=%-12s reach=%-5v class=%-4s durable=%s\n",
			t.Name, t.Type, t.State, t.Reachable, t.ClassHint, t.DurableID)
		fmt.Printf("       usage %s/%s (%.0f%%)  mount=%q dev=%q\n",
			gib(t.UsedBytes), gib(t.TotalBytes), t.UsedFraction*100, t.MountPath, t.BackingDevice)
		fmt.Printf("       smart: health=%s%s\n", t.Smart.Health, smartCounters(t.Smart))
		if t.ThinPool != nil {
			meta := "n/a"
			if t.ThinPool.MetadataUsedFraction != nil {
				meta = fmt.Sprintf("%.1f%%", *t.ThinPool.MetadataUsedFraction*100)
			}
			fmt.Printf("       thin-pool: data=%.1f%% metadata=%s\n", t.ThinPool.DataUsedFraction*100, meta)
		}
	}

	if watch <= 0 {
		fmt.Println("=== selftest=storage OK (observe pass; pass -watch D for the live watchdog) ===")
		return 0
	}

	// Live watchdog window: re-mount response active. Cycle a drive and watch the logs.
	fmt.Printf("  --- watching for %s (cycle a drive now; detect → report → re-mount) ---\n", watch)
	wctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()
	wctx, cancel2 := context.WithTimeout(wctx, watch)
	defer cancel2()

	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
	wd := storage.NewWatchdog(storage.WatchdogOptions{
		Targets:   storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
		Liveness:  storage.NewHostLiveness(hostReader, 0),
		Remounter: remounter,
		Trigger:   func() { logger.Info("storage: (selftest) would send out-of-band host-report now") },
		Interval:  cfg.Storage.WatchdogInterval(),
		Debounce:  cfg.Storage.WatchdogDebounce(),
		Logger:    logger,
	})
	_ = wd.Run(wctx)
	fmt.Println("=== selftest=storage watch window ended ===")
	return 0
}

// smartCounters renders the non-nil SMART counters compactly for the selftest table.
func smartCounters(s hub.SmartSummary) string {
	var parts []string
	add := func(name string, v *int) {
		if v != nil {
			parts = append(parts, fmt.Sprintf("%s=%d", name, *v))
		}
	}
	add("temp", s.TemperatureC)
	add("poh", s.PowerOnHours)
	add("realloc", s.ReallocatedSectors)
	add("pending", s.PendingSectors)
	add("offline_unc", s.OfflineUncorrectable)
	add("crit_warn", s.CriticalWarning)
	add("media_err", s.MediaErrors)
	add("pct_used", s.PercentageUsed)
	if len(parts) == 0 {
		return ""
	}
	return " " + strings.Join(parts, " ")
}

// runSelftestBackup runs one vzdump of -vmid to the configured local backup target and
// prints the resulting Backup record. Standalone (no hub). Live + benign.
func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if vmid == 0 {
		fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
		return 2
	}
	target := cfg.Backup.BackupTarget()
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, target)
	runner := backup.NewBackupRunner(px, target, "", "felhom selftest", "", logger)
	rec, err := runner.Backup(ctx, vmid)
	printJSON("backup record", rec)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] backup:", err)
		return 1
	}
	fmt.Printf("=== selftest=backup OK (crash-consistent=%v, archive=%s) ===\n", rec.CrashConsistent, rec.Archive)
	return 0
}

// runSelftestRestoreTest runs one self-restore-test (restore → net-link-down → boot → verify
// running → teardown) of -archive (or the newest backup on the local target) into a scratch
// guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
// crashed test is reaped before this run.
func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest=restore-test:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	// Build a journaled engine so a leaked scratch from a prior crash is recoverable.
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
	})

	fmt.Printf("=== felhom-agent %s selftest=restore-test ===\n", version)
	fmt.Println("  --- recover: reaping any leaked scratch from a prior crashed test ---")
	rec := engine.Recover(ctx)
	fmt.Printf("  recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)

	target := cfg.Backup.BackupTarget()
	if archive == "" {
		runner := backup.NewBackupRunner(px, target, "", "", "", logger)
		archive, err = runner.PickRestoreCandidate(ctx)
		if err != nil {
			fmt.Fprintln(os.Stderr, "  [FAIL] pick backup:", err)
			return 1
		}
		if archive == "" {
			fmt.Fprintln(os.Stderr, "  [FAIL] no backup available on", target, "(run --selftest=backup first)")
			return 1
		}
	}
	min, max := cfg.Backup.ScratchBand()
	fmt.Printf("  restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
	res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
		Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
		ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, target),
	})
	printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
	if res.Skipped {
		fmt.Println("=== selftest=restore-test SKIPPED (no free scratch VMID in band) ===")
		return 0
	}
	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] restore-test (scratch %d): %v\n", res.ScratchVMID, res.Err)
		return 1
	}
	fmt.Printf("=== selftest=restore-test OK (scratch %d restored+booted+verified+torn-down in %s) ===\n", res.ScratchVMID, res.Duration.Round(time.Second))
	return 0
}

// runSelftestBringUp runs the REAL journaled bring-up job (slice 7) on-demand: restore →
// reset identity → size → attach mounts → start link-up, then tears the guest down by default
// (a selftest must not leave a guest running) unless -keep. -mode picks provision (golden, fresh
// identity) or dr (customer backup, preserve continuity). It first Recovers, so a leaked guest
// from a prior crashed bring-up is reaped before this run.
// bringUpSizing carries the OS-rootfs / Docker-data sizing knobs from the CLI into a bring-up.
// Defaults (0/"") keep the golden's baked sizes; the provisioning spec sources these per-customer
// (flags now; the slice-10 hub storage manifest later — see bringup.go GuestMount comment).
type bringUpSizing struct {
	RootfsGrowGB  int
	DataVolGrowGB int
	DataVolMount  string
	SysDataGrowGB int
	SysDataMount  string
	Cores         int // 0 = keep golden default (CPU-core cap applied pre-start)
	MemoryMB      int // 0 = keep golden default (RAM cap in MiB, applied pre-start)
}

func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Logger, mode, archive string, vmid int, hostname string, keep bool, sizing bringUpSizing) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	var bmode reconcile.BringUpMode
	switch mode {
	case "provision":
		bmode = reconcile.ModeProvision
	case "dr":
		bmode = reconcile.ModeDRGuestLoss
	default:
		fmt.Fprintf(os.Stderr, "selftest=bring-up: -mode must be provision|dr (got %q)\n", mode)
		return 2
	}
	if archive == "" || vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=bring-up requires -archive <volid> and -vmid <N>")
		return 2
	}
	if cfg.Backup.RestoreStorage == "" {
		fmt.Fprintln(os.Stderr, "selftest=bring-up requires backup.restore_storage in config")
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
	})

	fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
	fmt.Println("  --- recover: reaping any half-built guest from a prior crashed bring-up ---")
	rec := engine.Recover(ctx)
	fmt.Printf("  recover: examined=%d bring_up_rolled_back=%d bring_up_clean=%d scratch_destroyed=%d\n",
		rec.Examined, rec.BringUpRolledBack, rec.BringUpClean, rec.ScratchDestroyed)

	spec := reconcile.BringUpSpec{
		Mode: bmode, Archive: archive, VMID: vmid, RestoreStorage: cfg.Backup.RestoreStorage,
		Hostname: hostname, KeepMAC: bmode == reconcile.ModeDRGuestLoss,
		Pool:  reconcile.DefaultPool, // restore into the felhom pool (both provision + DR); required under the scoped token
		Cores: sizing.Cores, MemoryMB: sizing.MemoryMB,
		RootfsGrowGB: sizing.RootfsGrowGB, DataVolGrowGB: sizing.DataVolGrowGB, DataVolMount: sizing.DataVolMount,
		SysDataGrowGB: sizing.SysDataGrowGB, SysDataMount: sizing.SysDataMount,
	}
	fmt.Printf("  bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
	res := engine.RunBringUp(ctx, spec)
	printJSON("bring-up record", res)

	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] bring-up (vmid %d): %v\n", vmid, res.Err)
		// On failure the job already compensating-rolled-back; nothing to tear down.
		return 1
	}
	fmt.Printf("  [OK] vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
	if len(res.StartWarnings) > 0 {
		fmt.Printf("  start warnings (recognized=%v): %v\n", res.WarningsRecognized, res.StartWarnings)
	}

	if keep {
		fmt.Printf("=== selftest=bring-up OK — guest %d KEPT (-keep) ===\n", vmid)
		return 0
	}
	// A selftest must not leave a guest running. Tear it down (out-of-band, like the spike).
	fmt.Printf("  --- teardown: destroying selftest guest %d ---\n", vmid)
	upid, err := px.DestroyLXC(ctx, vmid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy: %v (destroy vmid %d manually)\n", err, vmid)
		return 1
	}
	if upid != "" {
		if _, err := px.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
			fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy task: %v (check vmid %d)\n", err, vmid)
			return 1
		}
	}
	fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
	return 0
}

// provisionArgs bundles the --selftest=provision inputs.
type provisionArgs struct {
	archive     string
	vmid        int
	hostname    string
	customerID  string        // baked into bootstrap (the hub config-pull target)
	hubPassword string        // the customer's hub retrieval passphrase (SECRET) — baked into bootstrap
	sizing      bringUpSizing // OS-rootfs / Docker-data sizing for the bring-up half
}

// sanitizeHostname makes s a DNS-safe LXC hostname (RFC 1123 label-ish): lowercase, any run of
// invalid characters collapses to a single '-', leading/trailing '-' stripped, capped at 63 chars.
// Returns "" if nothing usable remains (caller then sets no hostname). PVE itself validates, but a
// customer id can legitimately contain characters (e.g. '_' or spaces) that a hostname cannot.
func sanitizeHostname(s string) string {
	s = strings.ToLower(strings.TrimSpace(s))
	var b strings.Builder
	prevDash := false
	for _, r := range s {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
			prevDash = false
		} else if !prevDash && b.Len() > 0 {
			b.WriteByte('-')
			prevDash = true
		}
	}
	out := strings.Trim(b.String(), "-")
	if len(out) > 63 {
		out = strings.Trim(out[:63], "-")
	}
	return out
}

// runSelftestProvision runs the FULL slice-8A provisioning chain on-demand: the slice-7 bring-up
// FRONT half (provision mode, golden) + the slice-8A BACK half (mint per-guest token → render the
// stable bootstrap.json → write 0600 → chown to the mapped guest-root → attach the read-only bind
// mount). The guest is KEPT (the golden's baked controller-bootstrap unit then deploys the baked
// controller from the mount). The per-guest token is NEVER printed (only its hash is persisted +
// the 0600 file holds the plaintext).
func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.Logger, a provisionArgs) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if a.archive == "" || a.vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=provision requires -archive <golden volid> and -vmid <N>")
		return 2
	}
	if cfg.Backup.RestoreStorage == "" {
		fmt.Fprintln(os.Stderr, "selftest=provision requires backup.restore_storage in config")
		return 2
	}
	if a.customerID == "" || a.hubPassword == "" {
		fmt.Fprintln(os.Stderr, "selftest=provision requires -customer-id and -hub-password (the customer's hub retrieval passphrase, so the controller pulls its config)")
		return 2
	}
	if err := cfg.LocalAPI.Validate(); err != nil || !cfg.LocalAPI.Enabled() {
		fmt.Fprintln(os.Stderr, "selftest=provision requires local_api.enable + local_api.listen_addr in config:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	// The leaf fingerprint baked into the bootstrap must be the SAME leaf the daemon's local-API
	// server serves — so use the configured (persisted) cert path. EnsureLeaf generates it once.
	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
	_, fingerprint, _, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest=provision: local-api leaf:", err)
		return 1
	}
	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest=provision: token store:", err)
		return 1
	}
	defer tokens.Close()

	// --- FRONT HALF: bring up the guest (provision mode) ---
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
	})

	// Default the guest hostname to the customer id (DNS-safe-sanitized) when not explicitly given,
	// so the CT/LXC is named meaningfully (e.g. "demo-felhom") instead of inheriting the golden's
	// baked "felhom-golden". An explicit -hostname always wins.
	hostname := a.hostname
	if hostname == "" {
		hostname = sanitizeHostname(a.customerID)
	}

	fmt.Printf("=== felhom-agent %s selftest=provision (vmid=%d customer=%s hostname=%s) ===\n", version, a.vmid, a.customerID, hostname)
	engine.Recover(ctx)
	fmt.Printf("  --- front half: bring-up (provision) %s → vmid %d ---\n", a.archive, a.vmid)
	res := engine.RunBringUp(ctx, reconcile.BringUpSpec{
		Mode: reconcile.ModeProvision, Archive: a.archive, VMID: a.vmid,
		RestoreStorage: cfg.Backup.RestoreStorage, Hostname: hostname,
		Pool:  reconcile.DefaultPool, // provision into the felhom pool; required under the scoped token
		Cores: a.sizing.Cores, MemoryMB: a.sizing.MemoryMB,
		RootfsGrowGB: a.sizing.RootfsGrowGB, DataVolGrowGB: a.sizing.DataVolGrowGB, DataVolMount: a.sizing.DataVolMount,
		SysDataGrowGB: a.sizing.SysDataGrowGB, SysDataMount: a.sizing.SysDataMount,
	})
	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)
		return 1
	}
	fmt.Printf("  [OK] front half: vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)

	// --- BACK HALF: mint token + populate the bootstrap config mount (host-side, no pct exec) ---
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	bh := provision.NewBackHalf(tokens, runner, "", logger)
	fmt.Println("  --- back half: mint per-guest token + populate bootstrap config mount ---")
	pres, err := bh.Provision(ctx, provision.Input{
		VMID:        a.vmid,
		Customer:    provision.DocCustomer{ID: a.customerID},
		Hub:         provision.DocHub{URL: cfg.Hub.URL, RetrievalPassword: a.hubPassword},
		Endpoint:    cfg.LocalAPI.ListenAddr,
		Fingerprint: fingerprint,
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] back-half provision (vmid %d): %v\n", a.vmid, err)
		return 1
	}
	fmt.Printf("  [OK] back half: bootstrap mount %s → %s on vmid %d (host dir %s)\n",
		pres.MountKey, pres.GuestPath, pres.VMID, pres.HostDir)
	fmt.Printf("  local-api endpoint %s · leaf fp %s · token: minted (not printed)\n", cfg.LocalAPI.ListenAddr, fingerprint)
	fmt.Printf("=== selftest=provision OK — guest %d provisioned + bootstrap-mounted (KEPT) ===\n", a.vmid)
	fmt.Println("    next: reboot the guest → the golden's baked controller-bootstrap unit deploys the controller,")
	fmt.Println("    which PULLS its controller.yaml from the hub (retrieval passphrase) and merges in this local_api.")
	return 0
}

// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
// blob. R is surfaced to stdout EXACTLY ONCE (never to the logger/journald). With -upload it PUTs
// the opaque blob to the hub. Enrollment-time, root-capable (reads the 0600 key).
func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath string) int {
	if storage == "" {
		storage = cfg.Escrow.PBSStorageID
	}
	if storage == "" {
		fmt.Fprintln(os.Stderr, "selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")
		return 2
	}
	keyPath := cfg.Backup.PBSEncKeyPath(storage)
	if _, err := os.Stat(keyPath); err != nil {
		fmt.Fprintf(os.Stderr, "selftest=escrow-create: PBS key for %q not found (%s): %v\n", storage, keyPath, err)
		return 1
	}

	// Slice 10D.1: optionally ALSO wrap the identity bundle under the same R, and carry the non-secret
	// directive for the hub. The bundle file is a 0600 secret (tunnel/pbs tokens); the directive is
	// non-secret (pbs repo/ns, expected fingerprint, tunnel id).
	var identity *escrow.IdentityBundle
	var directive json.RawMessage
	if identityBundlePath != "" {
		raw, err := os.ReadFile(identityBundlePath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "selftest=escrow-create: reading identity bundle %s: %v\n", identityBundlePath, err)
			return 1
		}
		var b escrow.IdentityBundle
		if err := json.Unmarshal(raw, &b); err != nil {
			fmt.Fprintf(os.Stderr, "selftest=escrow-create: identity bundle is not valid JSON {tunnel_token,pbs_token}: %v\n", err)
			return 1
		}
		identity = &b
		if directivePath != "" {
			if d, err := os.ReadFile(directivePath); err == nil && json.Valid(d) {
				directive = d
			}
		}
	}

	fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, storage, escrow.DefaultPosture, identity != nil)
	// NB: nothing about R is logged. The logger never sees R; only stdout does, once.
	logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath, "with_identity", identity != nil)

	R, res, err := escrow.Create(ctx, escrow.CreateOptions{
		KeyPath:         keyPath,
		Posture:         escrow.Posture(cfg.Escrow.Posture),
		WantOfflineCopy: offline,
		WantPaperkey:    paperkey,
		IdentityBundle:  identity,
	})
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] escrow create:", err)
		return 1
	}

	// Surface R EXACTLY ONCE — to stdout, with a write-it-down banner. Never logged/persisted.
	fmt.Println()
	fmt.Println("  ┌──────────────────────────────────────────────────────────────────────┐")
	fmt.Println("  │  RECOVERY CODE — write it down now. It is shown ONCE and never stored.  │")
	fmt.Println("  │  Without it your offsite backups are unrecoverable, by anyone.          │")
	fmt.Println("  └──────────────────────────────────────────────────────────────────────┘")
	fmt.Println("     " + R)
	fmt.Println()
	R = "" // drop our reference promptly

	fmt.Printf("  blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n",
		len(res.Blob), res.KeyFingerprint, res.Posture, res.EntropyBits)
	fmt.Println("  self-verify: the blob unwraps back to the key with R (recoverability confirmed)")

	if offline && len(res.OfflineCopy) > 0 {
		fmt.Println("  --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---")
		fmt.Println(base64.StdEncoding.EncodeToString(res.OfflineCopy))
	}
	if paperkey && res.Paperkey != "" {
		fmt.Println("  --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---")
		fmt.Println(res.Paperkey)
	}

	if len(res.IdentityBlob) > 0 {
		fmt.Printf("  identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", len(res.IdentityBlob))
	}
	if upload {
		if err := uploadEscrowBlob(ctx, cfg, res, directive); err != nil {
			fmt.Fprintln(os.Stderr, "  [FAIL] upload escrow to hub:", err)
			return 1
		}
		fmt.Println("  uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
	}
	fmt.Println("=== selftest=escrow-create OK ===")
	return 0
}

// runSelftestEscrowConsume exercises the slice-10C production Consume path live: recover K from an
// R-wrapped blob, gate it on the expected fingerprint, install it at -keydest. R is taken BY HAND
// from the env var FELHOM_RECOVERY_CODE (kept off the command line / ps) — never a flag, never
// logged. The other three inputs (blob/fingerprint/keydest) are flags (10D sources blob+fingerprint
// from the hub directive). This is the real Consume code, not a throwaway harness.
func runSelftestEscrowConsume(ctx context.Context, logger *slog.Logger, blobPath, expectedFP, keyDest string) int {
	if blobPath == "" || expectedFP == "" || keyDest == "" {
		fmt.Fprintln(os.Stderr, "selftest=escrow-consume requires -blob, -fingerprint and -keydest (R via env FELHOM_RECOVERY_CODE)")
		return 2
	}
	R := os.Getenv("FELHOM_RECOVERY_CODE")
	if R == "" {
		fmt.Fprintln(os.Stderr, "selftest=escrow-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input; never a flag/arg)")
		return 2
	}
	blob, err := os.ReadFile(blobPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "selftest=escrow-consume: reading blob %s: %v\n", blobPath, err)
		return 1
	}
	fmt.Printf("=== felhom-agent %s selftest=escrow-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
	logger.Info("escrow: consuming R-wrapped escrow (Unwrap → fingerprint-gate → install)",
		"blob_bytes", len(blob), "key_dest", keyDest) // R is NOT logged
	if err := escrow.Consume(ctx, blob, R, expectedFP, keyDest); err != nil {
		R = ""                                            // drop the reference
		fmt.Fprintln(os.Stderr, "  [FAIL] consume:", err) // the error never contains R or key bytes
		return 1
	}
	R = "" // drop the reference promptly
	fmt.Printf("  [OK] recovered key installed at %s (fingerprint-gated, 0600) — ready for the PBS restore\n", keyDest)
	fmt.Println("=== selftest=escrow-consume OK ===")
	return 0
}

// runSelftestIdentityConsume recovers the IDENTITY bundle from its age blob with R (slice 10D.1/10D.3)
// and writes the recovered {tunnel_token, pbs_token} JSON to -keydest (0600). R is taken BY HAND from
// FELHOM_RECOVERY_CODE (off the command line); the recovered tokens are never logged. The drill then
// uses the tunnel token to re-establish the tunnel + the pbs token for steady-state.
func runSelftestIdentityConsume(ctx context.Context, logger *slog.Logger, blobPath, keyDest string) int {
	if blobPath == "" || keyDest == "" {
		fmt.Fprintln(os.Stderr, "selftest=identity-consume requires -blob and -keydest (R via env FELHOM_RECOVERY_CODE)")
		return 2
	}
	R := os.Getenv("FELHOM_RECOVERY_CODE")
	if R == "" {
		fmt.Fprintln(os.Stderr, "selftest=identity-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input)")
		return 2
	}
	blob, err := os.ReadFile(blobPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "selftest=identity-consume: reading blob %s: %v\n", blobPath, err)
		return 1
	}
	fmt.Printf("=== felhom-agent %s selftest=identity-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
	logger.Info("escrow: recovering identity bundle from R-wrapped age blob", "blob_bytes", len(blob)) // R + tokens NOT logged
	bundle, err := escrow.UnwrapIdentityBundle(ctx, blob, R)
	if err != nil {
		R = ""
		fmt.Fprintln(os.Stderr, "  [FAIL] identity consume:", err) // never contains R or token bytes
		return 1
	}
	R = ""
	raw, _ := json.Marshal(bundle)
	if err := os.WriteFile(keyDest, raw, 0o600); err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] writing recovered bundle:", err)
		return 1
	}
	fmt.Printf("  [OK] identity recovered (tunnel_token + pbs_token) → %s (0600) — never printed\n", keyDest)
	fmt.Println("=== selftest=identity-consume OK ===")
	return 0
}

// escrowUploadRequest is the agent→hub wire shape for the opaque escrow blob. MUST stay in lockstep
// with the hub's ingest struct (felhom-hub api.escrowUploadRequest). The hub stores the bytes and
// never decrypts them.
type escrowUploadRequest struct {
	BlobB64        string `json:"blob_b64"`        // base64 of the opaque R-wrapped blob (ciphertext)
	KeyFingerprint string `json:"key_fingerprint"` // for operator display only
	Posture        string `json:"posture"`         // e.g. "zero_knowledge"
	// Slice 10D.1 — optional DR bundle (identity escrow + non-secret directive). Omitted in slice-7.
	IdentityBlobB64 string          `json:"identity_blob_b64,omitempty"`
	DirectiveJSON   json.RawMessage `json:"directive,omitempty"`
	CreatedAt       string          `json:"created_at"` // RFC3339
}

// uploadEscrowBlob PUTs the opaque blob (and, for 10D, the identity blob + non-secret directive) to
// the hub, authed with the per-host key. The hub stores ciphertext + non-secret fields; no usable
// secret leaves the agent.
func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateResult, directive json.RawMessage) error {
	if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
		return fmt.Errorf("hub not configured (url/host_id/api_key)")
	}
	upReq := escrowUploadRequest{
		BlobB64:        base64.StdEncoding.EncodeToString(res.Blob),
		KeyFingerprint: res.KeyFingerprint,
		Posture:        string(res.Posture),
		CreatedAt:      time.Now().UTC().Format(time.RFC3339),
	}
	if len(res.IdentityBlob) > 0 {
		upReq.IdentityBlobB64 = base64.StdEncoding.EncodeToString(res.IdentityBlob)
		upReq.DirectiveJSON = directive
	}
	body, _ := json.Marshal(upReq)
	url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
	req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+cfg.Hub.APIKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
	}
	return nil
}

// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
// path), then lists + prints the resulting PBSSnapshot records (verify-state included).
// Standalone on the host. Covers the runbook's (c) verify and (d) list.
func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version)
	targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] discover pbs storages:", err)
		return 1
	}
	if len(targets) == 0 {
		fmt.Println("  no pbs storages configured on this host")
		return 0
	}
	store := pbs.NewSnapshotStore()
	loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
		Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil },
		Store:   store,
		Logger:  logger,
	})
	// One synchronous verify+list pass over all datastores.
	loop.RunOnce(ctx)
	snaps := store.PBSSnapshots(ctx)
	printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps)
	failed := 0
	for _, s := range snaps {
		if s.VerifyState == pbs.VerifyFailed {
			failed++
		}
	}
	if failed > 0 {
		fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed)
		return 1
	}
	fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets))
	return 0
}

// printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
func printJSON(label string, v any) {
	if b, err := json.MarshalIndent(v, "  ", "  "); err == nil {
		fmt.Printf("  --- %s ---\n  %s\n", label, string(b))
	}
}

// runSelftestRead loads config, builds the API client, and runs the read-only
// queries against the live host, printing a short health report. It mutates
// nothing. Missing/invalid config is reported cleanly (no panic).
func runSelftestRead(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
		return 1
	}
	logger.Info("selftest (read-only) starting", "config", fmt.Sprintf("%+v", cfg.Redacted().Proxmox))

	client, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: client init:", err)
		return 1
	}

	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest (read-only) ===\n", version)
	fmt.Printf("endpoint : %s  node=%s\n", cfg.Proxmox.Endpoint, cfg.Proxmox.Node)

	fail := 0
	report := func(label string, err error) bool {
		if err != nil {
			fmt.Printf("  [FAIL] %-14s %v\n", label, err)
			fail++
			return false
		}
		return true
	}

	if v, err := client.Version(ctx); report("version", err) {
		fmt.Printf("  [ ok ] %-14s PVE %s (release %s)\n", "version", v.Version, v.Release)
	}
	if nodes, err := client.Nodes(ctx); report("nodes", err) {
		fmt.Printf("  [ ok ] %-14s %d node(s)\n", "nodes", len(nodes))
		for _, n := range nodes {
			marker := "  "
			if n.Node == cfg.Proxmox.Node {
				marker = "* "
			}
			fmt.Printf("           %s%s status=%s fp=%s…\n", marker, n.Node, n.Status, head(n.SSLFingerprint, 17))
		}
	}
	if s, err := client.NodeStatus(ctx); report("node status", err) {
		fmt.Printf("  [ ok ] %-14s up %s, load %v, mem %s/%s, root %s/%s\n", "node status",
			dur(s.Uptime), s.LoadAvg,
			gib(s.Memory.Used), gib(s.Memory.Total), gib(s.RootFS.Used), gib(s.RootFS.Total))
	}
	if gs, err := client.ListLXC(ctx); report("list lxc", err) {
		fmt.Printf("  [ ok ] %-14s %d guest(s)\n", "list lxc", len(gs))
		for _, g := range gs {
			fmt.Printf("           - %d %q status=%s\n", g.VMID, g.Name, g.Status)
		}
	}
	if ss, err := client.NodeStorage(ctx); report("storage", err) {
		fmt.Printf("  [ ok ] %-14s %d store(s)\n", "storage", len(ss))
		for _, s := range ss {
			fmt.Printf("           - %-10s type=%-8s content=%s used=%s/%s\n",
				s.Storage, s.Type, s.Content, gib(s.Used), gib(s.Total))
		}
	}

	if fail > 0 {
		fmt.Printf("=== selftest FAILED (%d check(s)) ===\n", fail)
		return 1
	}
	fmt.Println("=== selftest OK ===")
	return 0
}

// runSelftestTask exercises WaitTask on a reversible op against -vmid: snapshot ->
// rollback -> delete-snapshot. Explicitly gated; never runs under bare --selftest.
func runSelftestTask(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
		return 1
	}
	if vmid == 0 {
		fmt.Fprintln(os.Stderr, "selftest=task requires -vmid N (a guest safe to snapshot/rollback)")
		return 2
	}
	client, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: client init:", err)
		return 1
	}

	// Ctrl-C aborts the wait cleanly.
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	const snap = "felhom-selftest"
	steps := []struct {
		name string
		do   func() (string, error)
	}{
		{"snapshot", func() (string, error) { return client.Snapshot(ctx, vmid, snap, "felhom-agent selftest") }},
		{"rollback", func() (string, error) { return client.Rollback(ctx, vmid, snap) }},
		{"delete-snapshot", func() (string, error) { return client.DeleteSnapshot(ctx, vmid, snap) }},
	}
	fmt.Printf("=== felhom-agent selftest=task (vmid %d, snapshot %q) ===\n", vmid, snap)
	for _, st := range steps {
		upid, err := st.do()
		if err != nil {
			fmt.Printf("  [FAIL] %-16s %v\n", st.name, err)
			return 1
		}
		fmt.Printf("  ....   %-16s upid=%s\n", st.name, upid)
		status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
		if err != nil {
			fmt.Printf("  [FAIL] %-16s %v\n", st.name, err)
			return 1
		}
		fmt.Printf("  [ ok ] %-16s exitstatus=%s\n", st.name, status.ExitStatus)
	}

	// Reversible SetConfig exercise: this is the first live use of the VM.Config.*
	// privilege cluster. It round-trips the cosmetic `description` field (no runtime
	// effect, fully reversible) to prove SetConfig works under the scoped token
	// before slice 4's reconcile is built on top of it.
	if rc := selftestSetConfig(ctx, client, vmid); rc != 0 {
		return rc
	}

	fmt.Println("=== selftest=task OK ===")
	return 0
}

// selftestSetConfig performs a reversible write+revert of the LXC `description`
// field on vmid and asserts both land. Returns 0 on success, non-zero (with a
// printed [FAIL] line) on any failure — the caller stops on a non-zero return.
func selftestSetConfig(ctx context.Context, client *proxmox.Client, vmid int) int {
	// 1. Read current state; capture the original description (may be absent).
	cfg, err := client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig: %v\n", "setconfig", err)
		return 1
	}
	origDesc, origPresent, err := extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description: %v\n", "setconfig", err)
		return 1
	}
	// PVE normalizes `description` by appending a trailing newline on read, so all
	// comparisons here use the shared reconcile.NormDescription (strip trailing
	// newlines) and restores write the normalized original — otherwise an exact-match
	// check sees false drift. Same helper the slice-4 reconciler uses to normalize
	// description, so the quirk has one source of truth.
	origDesc = reconcile.NormDescription(origDesc)

	// 2. Write the marker.
	marker := "felhom-selftest " + time.Now().UTC().Format(time.RFC3339)
	if rc := applySetConfig(ctx, client, vmid, "setconfig", map[string]string{"description": marker}); rc != 0 {
		return rc
	}

	// 3. Verify the marker landed.
	cfg, err = client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig (verify): %v\n", "setconfig", err)
		return 1
	}
	got, present, err := extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description (verify): %v\n", "setconfig", err)
		return 1
	}
	if !present || reconcile.NormDescription(got) != marker {
		fmt.Printf("  [FAIL] %-16s write did not land: present=%v got=%q want=%q\n", "setconfig", present, got, marker)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s description verified == marker\n", "verify-write")

	// 4. Restore the original value (or clear it if it was absent originally).
	var revert map[string]string
	if origPresent {
		revert = map[string]string{"description": origDesc}
	} else {
		revert = map[string]string{"delete": "description"}
	}
	if rc := applySetConfig(ctx, client, vmid, "setconfig-revert", revert); rc != 0 {
		return rc
	}

	// 5. Confirm the restore.
	cfg, err = client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig (revert verify): %v\n", "setconfig-revert", err)
		return 1
	}
	got, present, err = extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description (revert verify): %v\n", "setconfig-revert", err)
		return 1
	}
	if origPresent {
		if !present || reconcile.NormDescription(got) != origDesc {
			fmt.Printf("  [FAIL] %-16s revert did not restore: present=%v got=%q want=%q\n", "setconfig-revert", present, reconcile.NormDescription(got), origDesc)
			return 1
		}
	} else if present {
		fmt.Printf("  [FAIL] %-16s revert did not clear: still present got=%q\n", "setconfig-revert", got)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s description restored to original\n", "verify-revert")
	return 0
}

// applySetConfig runs one SetConfig and asserts success, handling PVE's dual-mode
// return: a UPID means async (WaitTask + assert exitstatus OK); an empty string
// means PVE applied it synchronously (not an error — phase1-2/mutate.go contract).
func applySetConfig(ctx context.Context, client *proxmox.Client, vmid int, step string, params map[string]string) int {
	upid, err := client.SetConfig(ctx, vmid, params)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s %v\n", step, err)
		return 1
	}
	if upid == "" {
		// Synchronous path: empty UPID is a clean success.
		fmt.Printf("  [ ok ] %-16s synchronous  exitstatus=OK\n", step)
		return 0
	}
	status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
	if err != nil {
		fmt.Printf("  [FAIL] %-16s %s  %v\n", step, upid, err)
		return 1
	}
	if status.ExitStatus != "OK" {
		fmt.Printf("  [FAIL] %-16s %s  exitstatus=%s\n", step, upid, status.ExitStatus)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s %s  exitstatus=%s\n", step, upid, status.ExitStatus)
	return 0
}

// extraString reads a string-valued key from GuestConfig.Extra (raw JSON). It
// returns ("", false, nil) when the key is absent, and decodes the JSON string
// otherwise.
func extraString(cfg proxmox.GuestConfig, key string) (string, bool, error) {
	raw, ok := cfg.Extra[key]
	if !ok || len(raw) == 0 {
		return "", false, nil
	}
	var s string
	if err := json.Unmarshal(raw, &s); err != nil {
		return "", false, err
	}
	return s, true, nil
}

// --- small helpers / flag type ---

func envOr(key, def string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return def
}

func head(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n]
}

func dur(seconds int64) string { return (time.Duration(seconds) * time.Second).String() }

func gib(bytes int64) string { return fmt.Sprintf("%.1fGiB", float64(bytes)/(1<<30)) }

// runSelftestLANResolver ensures dnsmasq + the host base config, then applies the split-horizon record
// for ONE guest (its live LAN IP + domain discovered from the running guest). Prints what it wrote;
// verify the actual resolution out-of-band (dig @<host-ip> <app>.<domain>). Requires a host LAN IP
// (lan_resolver.host_ip or derivable from local_api.listen_addr).
func runSelftestLANResolver(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=lanresolver: -vmid is required")
		return 1
	}
	lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
	if lr.HostIP == "" {
		fmt.Fprintln(os.Stderr, "selftest=lanresolver: no host IP (set lan_resolver.host_ip or local_api.listen_addr)")
		return 1
	}
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	mgr := lanresolver.NewManager(runner, lr.HostIP, lr.Upstreams, logger)

	fmt.Printf("=== felhom-agent %s selftest=lanresolver (vmid=%d host_ip=%s) ===\n", version, vmid, lr.HostIP)
	if err := mgr.EnsureDnsmasq(ctx); err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] EnsureDnsmasq: %v\n", err)
		return 1
	}
	fmt.Printf("  [OK] dnsmasq present + base config (listen %s, upstreams %v)\n", lr.HostIP, lr.Upstreams)
	cid := lanresolver.CustomerID(lr.StateDir, vmid)
	if err := mgr.ReconcileGuest(ctx, vmid, cid); err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] ReconcileGuest: %v\n", err)
		return 1
	}
	fmt.Printf("  [OK] split-horizon applied for guest %d (customer=%q). Verify: dig @%s felhom.<domain>\n", vmid, cid, lr.HostIP)
	return 0
}

// selftestFlag is a flag.Value that also satisfies IsBoolFlag, so `--selftest`
// works bare (read-only) and `--selftest=task` / `--selftest=read` set the mode.
type selftestFlag struct{ mode string }

func (f *selftestFlag) String() string   { return f.mode }
func (f *selftestFlag) IsBoolFlag() bool { return true }
func (f *selftestFlag) Set(v string) error {
	switch v {
	case "true", "", "read":
		f.mode = "read"
	case "task":
		f.mode = "task"
	case "hub":
		f.mode = "hub"
	case "storage":
		f.mode = "storage"
	case "backup":
		f.mode = "backup"
	case "restore-test":
		f.mode = "restore-test"
	case "pbs-verify":
		f.mode = "pbs-verify"
	case "lanresolver":
		f.mode = "lanresolver"
	case "bring-up":
		f.mode = "bring-up"
	case "provision":
		f.mode = "provision"
	case "escrow-create":
		f.mode = "escrow-create"
	case "escrow-consume":
		f.mode = "escrow-consume"
	case "identity-consume":
		f.mode = "identity-consume"
	case "controller-swap":
		f.mode = "controller-swap"
	default:
		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap)", v)
	}
	return nil
}