felhom-agent/cmd/felhom-agent/main.go

// Command felhom-agent is the host agent.
//
// With no --selftest flag it runs as the daemon: the host-report poll loop
// (slice 3) that periodically POSTs a read-only host-report to the hub (the
// heartbeat). --selftest=read|task exercise the proxmox layer; --selftest=hub does
// one collect+report against the hub and prints what it would send.
package main

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"net/netip"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
	"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
	"gitea.dooplex.hu/admin/felhom-agent/internal/config"
	"gitea.dooplex.hu/admin/felhom-agent/internal/desired"
	"gitea.dooplex.hu/admin/felhom-agent/internal/dr"
	"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
	"gitea.dooplex.hu/admin/felhom-agent/internal/fasttick"
	"gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd"
	"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
	"gitea.dooplex.hu/admin/felhom-agent/internal/guestnet"
	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
	"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
	"gitea.dooplex.hu/admin/felhom-agent/internal/mgmtplane"
	"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
	"gitea.dooplex.hu/admin/felhom-agent/internal/pbsdr"
	"gitea.dooplex.hu/admin/felhom-agent/internal/poke"
	"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
	"gitea.dooplex.hu/admin/felhom-agent/internal/selfheal"
	"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
	"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
)

// version is the agent version. Overridable at build time with
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
var version = "0.92.1"

// runGuestHook is the PVE hook body (`felhom-agent guest-hook <vmid> <phase>`). On pre-start it
// creates placeholder dirs for any absent bind-mount source so the guest always boots (the C1 net);
// on post-start it re-arms idle NAS automount triggers so the fresh guest namespace sees network
// shares again (RCA fix 1 — a new namespace inherits real mounts but not idle autofs triggers).
// It ALWAYS returns cleanly (exit 0) — a hook must never block a guest start. Output is written to
// stderr (PVE captures hook output into the task log).
func runGuestHook(args []string) {
	if len(args) < 2 {
		return
	}
	vmid, phase := args[0], args[1]
	switch phase {
	case guesthook.PhasePreStart:
		// F10/rc255 (CAMPAIGN-3): a pre-start hook that panics or hangs would BLOCK the guest start (the
		// campaign saw a guest bricked while a unit sat start-limited). Every phase runs recover-wrapped
		// under a hard timeout and this function ALWAYS returns cleanly (main then exits 0).
		runHookPhase(vmid, "pre-start", 30*time.Second, func(context.Context) {
			created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
			if len(created) > 0 {
				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
			}
			if err != nil {
				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
			}
		})
	case guesthook.PhasePostStart:
		runHookPhase(vmid, "post-start", 30*time.Second, func(ctx context.Context) {
			postStartNetworkReassertFn(ctx, vmid)
		})
	}
}

// runHookPhase runs one guest-hook phase body under a hard timeout with a panic recover, so NO phase
// can ever fail the guest start (CAMPAIGN-3 F10/rc255). A panic is logged to the PVE task log (stderr)
// and swallowed; a body that overruns the timeout is abandoned (its context is cancelled) while the
// hook returns. This is the Go belt; the wrapper script's `|| true; exit 0` is the shell belt.
func runHookPhase(vmid, phase string, timeout time.Duration, body func(context.Context)) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	done := make(chan struct{})
	go func() {
		defer func() {
			if r := recover(); r != nil {
				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s PANIC recovered (boot continues): %v\n", vmid, phase, r)
			}
			close(done)
		}()
		body(ctx)
	}()
	select {
	case <-done:
	case <-ctx.Done():
		fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s timed out after %s (boot continues)\n", vmid, phase, timeout)
	}
}

// postStartNetworkReassertFn is the post-start hook action (seam — tests assert the wiring without
// touching real systemctl).
var postStartNetworkReassertFn = guesthook.PostStartNetworkReassert

func main() {
	// Pre-start self-heal hook entrypoint. PVE invokes the registered hookscript as
	// `<bin> guest-hook <vmid> <phase>`. Handled BEFORE flag parsing — it takes positional args, must be
	// fast, needs no config/daemon/network, and must NEVER exit nonzero (a hook that fails would block
	// the guest start). See internal/guesthook (the C1 net).
	if len(os.Args) >= 2 && os.Args[1] == "guest-hook" {
		runGuestHook(os.Args[2:])
		return
	}
	var (
		cfgPath       string
		selftest      selftestFlag
		vmid          int
		watch         time.Duration
		archive       string
		mode          string
		hostname      string
		keep          bool
		rootfsGrow    int
		dataVolGrow   int
		dataVolMount  string
		sysDataGrow   int
		sysDataMount  string
		cores         int
		memoryMB      int
		pbsStorage    string
		paperkey      bool
		offline       bool
		upload        bool
		custID        string
		custDomain    string
		custName      string
		custEmail     string
		hubPassword   string
		blobPath      string
		expectedFP    string
		keyDest       string
		installWGKey  bool
		idBundlePath  string
		directivePath string
		swapImage     string
		outputMode    string
		showVersion   bool
	)
	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `restore-test-due` = READ-ONLY: print the per-tier due verdict the scheduler would act on, with its cost; `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-cores/-memory (-sysdata-grow is deprecated: folded into -datavol-grow); keeps the guest)")
	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
	flag.StringVar(&mode, "mode", "provision", "for --selftest=bring-up: `provision` (golden, fresh identity) | `dr` (customer backup, preserve continuity)")
	flag.StringVar(&hostname, "hostname", "", "for --selftest=bring-up provision: the hostname to set on the new guest")
	flag.BoolVar(&keep, "keep", false, "for --selftest=bring-up: KEEP the guest instead of tearing it down at the end")
	flag.IntVar(&rootfsGrow, "rootfs-grow", 0, "for --selftest=bring-up|provision: grow the OS rootfs by this many GiB after restore (0 = keep golden size)")
	flag.IntVar(&dataVolGrow, "datavol-grow", 0, "for --selftest=bring-up|provision: grow the golden's Docker-data volume (mp0) by this many GiB (0 = keep golden size)")
	flag.StringVar(&dataVolMount, "datavol-mount", "", "for --selftest=bring-up|provision: the mpN slot of the Docker-data volume to grow (default mp0)")
	// R-165: the second volume is gone (build-golden.sh v3.0.0 ships ONE). These two flags are kept
	// ACCEPTED because felhom-host-install.sh passes -sysdata-grow and an installer and an agent do not
	// upgrade in the same instant — removing them would make every install fail on an unknown flag.
	// -sysdata-grow is NOT inert: its GiB are folded into the single volume's grow (bringup.go 4b), so
	// an old installer still produces the same total capacity. -sysdata-mount selects nothing.
	flag.IntVar(&sysDataGrow, "sysdata-grow", 0, "DEPRECATED (R-165): there is one data volume now; this value is ADDED to -datavol-grow rather than growing a second volume. Kept so an older felhom-host-install.sh keeps working")
	flag.StringVar(&sysDataMount, "sysdata-mount", "", "DEPRECATED (R-165): ignored — there is no second volume to select")
	flag.IntVar(&cores, "cores", 0, "for --selftest=bring-up|provision: cap the guest to N CPU cores (0 = keep golden default). Applied in the pre-start config PUT.")
	flag.IntVar(&memoryMB, "memory", 0, "for --selftest=bring-up|provision: cap the guest RAM to N MiB (0 = keep golden default). Applied pre-start.")
	flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
	flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
	flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
	flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
	flag.StringVar(&blobPath, "blob", "", "for --selftest=escrow-consume: path to the R-wrapped escrow blob file")
	flag.StringVar(&expectedFP, "fingerprint", "", "for --selftest=escrow-consume: the EXPECTED key fingerprint (the gate target)")
	flag.StringVar(&keyDest, "keydest", "", "for --selftest=escrow-consume: where to install the recovered key (0600)")
	flag.BoolVar(&installWGKey, "install-wg-key", false, "for --selftest=identity-consume: ALSO install the recovered wg_private_key into wgtunnel's key file (S5 DR; create-only, refuses to overwrite)")
	flag.StringVar(&idBundlePath, "identity-bundle", "", "for --selftest=escrow-create: a 0600 JSON file {tunnel_token,pbs_token} to ALSO escrow under R (10D)")
	flag.StringVar(&directivePath, "directive", "", "for --selftest=escrow-create: a JSON file with the non-secret DR directive (pbs repo/ns, expected fingerprint, tunnel id)")
	flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id — the hub config-pull target, baked into the guest's bootstrap")
	flag.StringVar(&hubPassword, "hub-password", "", "for --selftest=provision: the customer's hub RETRIEVAL PASSPHRASE (SECRET) — baked into bootstrap.json so the controller pulls its config (and the customer-scoped hub key) from the hub. The customer must already exist in the hub.")
	flag.StringVar(&swapImage, "image", "", "for --selftest=controller-swap: the target controller image ref (gitea.dooplex.hu/admin/felhom-controller:<semver>) — must already be pulled in the guest")
	flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)")
	flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)")
	flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)")
	flag.StringVar(&outputMode, "output", "text", "for --selftest=escrow-create: `text` (human, default — unchanged) | `json` (one machine-readable JSON object on stdout carrying the recovery code; every human line to stderr; the controller-driven ceremony's parse surface)")
	flag.BoolVar(&showVersion, "version", false, "print version and exit")
	flag.Parse()

	if showVersion {
		fmt.Println("felhom-agent", version)
		return
	}

	cfg, err := config.Load(cfgPath)
	if err != nil {
		// A missing default config file is fine if env provides the values; only a
		// present-but-unreadable/invalid file is fatal here.
		if !(os.IsNotExist(errors.Unwrap(err)) && cfgPath == flag.Lookup("config").DefValue) {
			fmt.Fprintln(os.Stderr, "config error:", err)
			os.Exit(2)
		}
		cfg = config.Default()
	}
	// logRing is the always-DEBUG capture ring (v0.83.0 observability): served by the
	// local API's GET /debug/logs and the heartbeat log-pull. Selftests ignore it.
	logger, logRing := applog.New(cfg.LogLevel)

	switch selftest.mode {
	case "":
		os.Exit(runDaemon(cfg, logger, logRing))
	case "read":
		os.Exit(runSelftestRead(context.Background(), cfg, logger))
	case "task":
		os.Exit(runSelftestTask(context.Background(), cfg, logger, vmid))
	case "hub":
		os.Exit(runSelftestHub(context.Background(), cfg, logger))
	case "storage":
		os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
	case "backup":
		os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
	case "restore-test":
		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
	case "restore-test-due":
		os.Exit(runSelftestRestoreTestDue(context.Background(), cfg, logger))
	case "pbs-verify":
		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
	case "lanresolver":
		os.Exit(runSelftestLANResolver(context.Background(), cfg, logger, vmid))
	case "wgtunnel":
		os.Exit(runSelftestWGTunnel(context.Background(), cfg, logger))
	case "bring-up":
		os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep,
			bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
				SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB}))
	case "provision":
		os.Exit(runSelftestProvision(context.Background(), cfg, logger, provisionArgs{
			archive: archive, vmid: vmid, hostname: hostname,
			customerID: custID, hubPassword: hubPassword,
			sizing: bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
				SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB},
		}))
	case "escrow-create":
		os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath, outputMode))
	case "escrow-consume":
		os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest))
	case "identity-consume":
		os.Exit(runSelftestIdentityConsume(context.Background(), cfg, logger, blobPath, keyDest, installWGKey))
	case "controller-swap":
		os.Exit(runSelftestControllerSwap(context.Background(), cfg, logger, vmid, swapImage))
	}
}

// runSelftestWGTunnel is the supervised single-shot bring-up (S3): keygen/register (marker-
// gated), one desired-state fetch, one apply, then the status stanza. Runs regardless of
// wg_tunnel.enabled (the operator invoked it deliberately) but uses the config's state
// dir/runner exactly as the daemon would.
func runSelftestWGTunnel(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	fmt.Printf("=== felhom-agent %s selftest=wgtunnel ===\n", version)
	wt := cfg.WGTunnel.WithDefaults()
	client, err := hub.NewClient(cfg.Hub, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] hub client:", err)
		return 1
	}
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	mgr := wgtunnel.NewManager(runner, client, wt.StateDir, logger)

	// One registration/adopt pass with no desired data, then one fetch + apply.
	mgr.Apply(ctx, false, nil)
	var fetched bool
	var block *hub.WireWireguard
	if resp, err := client.FetchDesiredState(ctx); err != nil {
		fmt.Fprintln(os.Stderr, "  [WARN] desired-state fetch failed (apply skipped):", err)
	} else {
		fetched = true
		block = resp.DesiredState.Wireguard
		fmt.Printf("  desired-state generation=%d wireguard-block=%v\n", resp.Generation, block != nil)
	}
	mgr.Apply(ctx, fetched, block)

	st := mgr.Status(ctx)
	fmt.Printf("  status: pubkey=%s registered=%v active=%v assigned_ip=%s", st.Pubkey, st.Registered, st.Active, st.AssignedIP)
	if st.LastHandshakeAgeS != nil {
		fmt.Printf(" handshake_age_s=%d", *st.LastHandshakeAgeS)
	}
	fmt.Println()
	if st.Registered && (block == nil || st.Active) {
		fmt.Println("  [OK] wgtunnel selftest complete")
		return 0
	}
	if !st.Registered {
		fmt.Println("  [WARN] not registered (hub unreachable or endpoint unset) — see log above")
	}
	return 0
}

// newProxmoxClient builds the read-path proxmox client from config.
func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) {
	return proxmox.NewClient(proxmox.Config{
		Endpoint: cfg.Proxmox.Endpoint,
		Node:     cfg.Proxmox.Node,
		Token:    cfg.Proxmox.Token,
		TLS: proxmox.TLSConfig{
			CAFile:             cfg.Proxmox.TLS.CAFile,
			Fingerprint:        cfg.Proxmox.TLS.Fingerprint,
			InsecureSkipVerify: cfg.Proxmox.TLS.InsecureSkipVerify,
		},
	})
}

// newHostOps builds the privileged storage surface (slice 5 Phase B) from config. It shells
// out via the same fenced Runner the proxmox layer uses (sudo -n, arg vectors, no shell);
// every argument is validated in internal/storage before any command is built. A
// missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a
// crash.
// Returns the concrete *storage.SudoHostOps (not the HostOps interface) so callers that need the
// methods outside that lean interface — the host-reboot mount re-assert and the Part-A1 network-mount
// surface — can reach them without a type assertion. It still satisfies storage.HostOps everywhere.
func newHostOps(cfg config.Config, logger *slog.Logger) *storage.SudoHostOps {
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	return storage.NewSudoHostOps(storage.SudoHostOpsConfig{
		Runner: runner,
		Bins: storage.Binaries{
			Systemctl: cfg.Privileged.Systemctl,
			Install:   cfg.Privileged.Install,
			Smartctl:  cfg.Privileged.Smartctl,
			Lvs:       cfg.Privileged.Lvs,
		},
		UnitDir:  cfg.Privileged.UnitDir,
		StageDir: cfg.Privileged.StageDir,
		Logger:   logger,
	})
}

// gateRemounter is the watchdog's benign re-mount response: it routes the re-mount through
// the reversibility gate (classified benign) and, if allowed, calls HostOps.EnsureMount. It
// lives here (not in internal/storage) so storage stays decoupled from reconcile — main is
// the one place that holds both.
type gateRemounter struct {
	gate   *reconcile.Gate
	ops    storage.HostOps
	hostID string
	logger *slog.Logger
}

// Remount authorizes (benign) and performs a by-UUID re-mount of a returned target.
func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
	dec := r.gate.Authorize(reconcile.IntentForStorageMount(r.hostID, t.Name), nil)
	if !dec.Allowed {
		r.logger.Warn("storage: re-mount refused by gate (unexpected for a benign mount)",
			"target", t.Name, "reason", dec.Reason)
		return
	}
	uuid := t.UUID
	if uuid == "" {
		uuid = strings.TrimPrefix(t.DurableID, "uuid:") // durable_id carries it for usb/local-dir
	}
	spec := storage.MountSpec{Name: t.Name, UUID: uuid, Where: t.MountPath}
	if err := r.ops.EnsureMount(ctx, spec); err != nil {
		r.logger.Error("storage: re-mount failed", "target", t.Name, "where", t.MountPath, "err", err)
		return
	}
	r.logger.Info("storage: re-mounted returned target", "target", t.Name, "where", t.MountPath)
}

// poolReadStatus probes the felhom-pool membership read (GET /pools/felhom) the stale-lock reaper's
// ownership scoping depends on (v0.62.0, audit A1). Needs `Pool.Audit` at `/pool/felhom` — granted
// by host-install v1.9.0; on an older ACL this reports degraded and the reaper fail-safes (skips).
func poolReadStatus(ctx context.Context, px *proxmox.Client) capability.Status {
	s := capability.Status{
		Name:     "pve:pool-read",
		Feature:  "stale-lock recovery scoping (pool ownership check)",
		Critical: false,
		Status:   capability.StatusOK,
	}
	if px == nil {
		s.Status, s.Reason = capability.StatusDegraded, "not configured"
		return s
	}
	pctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	if _, err := px.Pool(pctx, reconcile.DefaultPool); err != nil {
		s.Status, s.Reason = capability.StatusDegraded, err.Error()
	}
	return s
}

// storeGrantStatuses probes whether the agent's OWN TOKEN may read the storages this box depends
// on — one capability.Status per configured backup tier (R-185).
//
// ── WHY THIS EXISTS, AND WHY IT IS NOT A CONTENT LISTING ─────────────────────────────────────
//
// On demo-felhom the token had FelhomAgentStore on local, local-lvm and felhom-pbs — and NOT on
// `felhom-backup`, the storage the same installer had configured as `local_backup_target`. Asked
// for that storage's content the API answers `{"data":[]}` while root sees three archives.
//
// **An empty listing is what a FORBIDDEN tier and a NEWBORN tier both return**, and no care at that
// call site can separate them: `pickForThisRun` skips an empty tier (correctly — a fresh offsite
// tier legitimately has nothing) and says "no settled archive yet". So the host tier on that box was
// never restore-testable and nothing ever mentioned it. That is this project's own rule failing in a
// new place: an empty answer is not evidence that there is nothing there.
//
// The permission question, unlike the listing, has a DEFINITE answer — so it is asked directly.
//
// ── WHAT IS PROBED, AND WHY NOT A FIXED LIST ─────────────────────────────────────────────────
//
// The tiers come from this box's own config (`BackupTiers()`), because a hardcoded probe list is
// precisely the defect being fixed — the installer's hardcoded ACL set is what drifted from the
// target it went on to configure. Probing what the box says it depends on cannot drift from it.
//
// CRITICAL, deliberately: a tier the agent cannot read is a tier whose backups are invisible to it
// and which is never restore-tested. The hub alerts only on Critical, and a non-critical entry here
// would ride the report and alert nobody — the same silence with extra steps.
//
// One exception, so an ordinary configuration is not turned into an alarm: a box with no dedicated
// target (`local_backup_target: "local"`, which host-install's own comment calls the DEGRADED
// fallback) is not treated as critical for that tier — see storeGrantCritical.
func storeGrantStatuses(ctx context.Context, px *proxmox.Client, cfg config.Config, repair *storeGrantRepairer) []capability.Status {
	tiers, _ := cfg.Backup.BackupTiers() // warnings are logged where the tiers are armed
	out := make([]capability.Status, 0, len(tiers))
	for _, t := range tiers {
		out = append(out, storeGrantStatus(ctx, px, t.TargetID, storeGrantCritical(t.TargetID), repair))
	}
	return out
}

// storeGrantRepairReportWindow is how long after a repair the capability keeps reporting the
// transition. It MUST exceed the hub report interval, or the record never reaches the operator.
//
// FOUND BY THE LIVE RUN, NOT BY THE TESTS (2026-08-04). The first implementation reported degraded
// for exactly "one cycle" — the probe call that did the repair. But `probeAll` is invoked
// INDEPENDENTLY by the startup/periodic self-check log and by the collector building a host report,
// so the repairing call was the LOG's, and the report built three seconds later found the grant
// present and reported `ok`. The agent's journal had the record; the hub had nothing; the operator
// would have learned nothing. That is precisely the silence R-190 is about, re-created inside its own
// mitigation.
//
// A latch on TIME rather than on call count fixes it: 20 minutes comfortably exceeds the 900 s report
// interval, so at least one host-report must carry the transition, and it still clears on its own.
const storeGrantRepairReportWindow = 20 * time.Minute

// storeGrantRepairMinInterval bounds how often a single tier's grant may be re-granted (Scenario F).
//
// A storage can be unreadable for reasons an ACL cannot fix — the storage is gone, PVE is wedged,
// the wrapper is missing. Without a bound the probe would re-grant on every report cycle forever: a
// repair loop is a new defect wearing a fix's clothes. One attempt per tier per hour is frequent
// enough that a real loss is repaired within one backup window, and rare enough that a permanent
// fault produces attempts you can count on one hand per day.
const storeGrantRepairMinInterval = time.Hour

// storeGrantRepairer bounds and records the self-repair. It is deliberately in-memory: an agent
// restart re-arms the repair, which is correct — a restart is exactly when a box should re-check
// everything it depends on.
type storeGrantRepairer struct {
	run      func(ctx context.Context, name string, args ...string) ([]byte, []byte, error)
	log      *slog.Logger
	mu       sync.Mutex
	last     map[string]time.Time // target id → last ATTEMPT (success or failure)
	repaired map[string]time.Time // target id → last CONFIRMED repair (drives the report latch)
}

// noteRepaired latches a confirmed repair so it is reported for storeGrantRepairReportWindow.
func (r *storeGrantRepairer) noteRepaired(target string, now time.Time) {
	if r == nil {
		return
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.repaired == nil {
		r.repaired = map[string]time.Time{}
	}
	r.repaired[target] = now
}

// recentlyRepaired reports whether a confirmed repair is still inside its report window — the latch
// that guarantees a host-report carries the transition even though the probe that repaired may have
// been a log-only one.
func (r *storeGrantRepairer) recentlyRepaired(target string, now time.Time) bool {
	if r == nil {
		return false
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	t, ok := r.repaired[target]
	return ok && now.Sub(t) < storeGrantRepairReportWindow
}

// mayAttempt reports whether a repair may run now for this target, and records the attempt if so.
func (r *storeGrantRepairer) mayAttempt(target string, now time.Time) bool {
	if r == nil || r.run == nil {
		return false
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.last == nil {
		r.last = map[string]time.Time{}
	}
	if t, ok := r.last[target]; ok && now.Sub(t) < storeGrantRepairMinInterval {
		return false
	}
	r.last[target] = now
	return true
}

// repair runs the EXISTING root wrapper's `grant` verb for this storage. It adds no privileged
// surface: `felhom-backup-target-apply grant *` is already in the sudoers allowlist for any storage
// id (configs/felhom-agent.sudoers), and the verb already grants BOTH the user and the token — a
// privsep token's rights are the intersection, so granting one of the two grants nothing usable.
//
// This is the pbsdr shape (internal/pbsdr/manager.go, the R-22 self-grant): on a refusal, run the
// root wrapper and RE-READ ONCE rather than dead-locking. Its restraint is copied too — one attempt,
// one confirmation, and anything still wrong stays loudly wrong.
func (r *storeGrantRepairer) repair(ctx context.Context, target string) error {
	rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()
	_, errOut, err := r.run(rctx, localapi.BackupTargetWrapperPath, "grant", target)
	if err != nil {
		r.log.Error("store-grant: SELF-REPAIR FAILED — the tier stays unreadable",
			"target", target, "err", err, "stderr", strings.TrimSpace(string(errOut)))
		return err
	}
	return nil
}

// storeGrantRequiredPriv is the privilege whose ABSENCE was measured to blind the content listing.
//
// Measured on demo-felhom 2026-08-03: the two storages that list through the token hold
// Datastore.Allocate + Datastore.AllocateSpace (the FelhomAgentStore role); the one that answers
// empty holds only what the box-wide grant propagates (Sys.Audit, SDN.Use, Datastore.Audit). It is
// NOT Datastore.Audit that is missing — checking for that would report the blinded storage healthy.
const storeGrantRequiredPriv = "Datastore.AllocateSpace"

// storeGrantCritical decides whether a missing grant on this target is Critical (operator-paged).
//
// "local" is host-install's DEGRADED fallback target — a box with no dedicated backup storage is a
// known, ordinary configuration, and turning it into a critical alert is how a signal becomes
// something an operator archives unread. It is still probed and still reported; only the paging
// differs.
func storeGrantCritical(targetID string) bool { return targetID != "local" }

// storeGrantStatus is one tier's grant probe. It NEVER reports ok when it could not ask: a
// self-check that fails open is worse than none, because it converts "I do not know" into "fine".
func storeGrantStatus(ctx context.Context, px *proxmox.Client, targetID string, critical bool, repair *storeGrantRepairer) capability.Status {
	s := capability.Status{
		Name:     "pve:store-grant:" + targetID,
		Feature:  "backup tier " + targetID + " readable by the agent (archive listing, restore-test candidacy)",
		Critical: critical,
		Status:   capability.StatusOK,
	}
	if px == nil {
		s.Status, s.Reason = capability.StatusDegraded, "not configured"
		return s
	}
	if targetID == "" {
		s.Status, s.Reason = capability.StatusDegraded, "tier has no target id"
		return s
	}
	pctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	privs, err := px.Permissions(pctx, "/storage/"+targetID)
	s = storeGrantVerdict(targetID, critical, privs, err)
	if err != nil {
		return s
	}
	if s.Status != capability.StatusDegraded {
		// Healthy — but if this tier was repaired moments ago, keep REPORTING the transition until a
		// host-report has certainly carried it. Without this latch the repairing probe may be a
		// log-only one and the hub never learns anything happened (measured live, see the window's
		// comment).
		return storeGrantHealthyVerdict(targetID, critical, s, repair.recentlyRepaired(targetID, time.Now()))
	}

	// ── R-190 mitigation: the grant is missing — repair it, and SAY that it was missing ──────────
	//
	// R-190 is a grant that demonstrably worked at 04:44 and was gone by 09:24, with a reinstall,
	// logged pveum activity and cluster-log entries all ruled out. The cause is still open; the
	// resilience does not have to wait for it. Everything needed already exists — the root wrapper,
	// its sudoers vector for any storage id, and the exact command — and until now the `grant` verb
	// had only ever been called at CREATION. That is the "built but never wired" shape, in a verb
	// rather than a seam.
	if !repair.mayAttempt(targetID, time.Now()) {
		// Bounded (Scenario F): an earlier attempt did not hold and it is too soon to try again. Stay
		// degraded and say why — a quiet "we already tried" is how a permanent fault becomes silence.
		s.Reason = "the agent token lacks " + storeGrantRequiredPriv + " on /storage/" + targetID +
			" and a self-repair was attempted within the last " + storeGrantRepairMinInterval.String() +
			" without holding — NOT retrying yet; this needs a human"
		return s
	}
	if rerr := repair.repair(ctx, targetID); rerr != nil {
		s.Reason = "the agent token lacks " + storeGrantRequiredPriv + " on /storage/" + targetID +
			" and the self-repair FAILED (" + rerr.Error() + ") — this tier's archives are INVISIBLE to the agent"
		return s // Scenario E: a failed repair must never mask the degraded state.
	}
	// Re-read ONCE to confirm, exactly as pbsdr does — the wrapper reporting success is a claim about
	// its own write; the grant being readable is a different claim, and it is the one that matters.
	cctx, ccancel := context.WithTimeout(ctx, 10*time.Second)
	defer ccancel()
	privs2, err2 := px.Permissions(cctx, "/storage/"+targetID)
	if err2 != nil || privs2[storeGrantRequiredPriv] != 1 {
		s.Reason = "the agent token lacks " + storeGrantRequiredPriv + " on /storage/" + targetID +
			" and the self-repair did not take (re-read says it is still missing) — this needs a human"
		return s
	}

	// REPAIRED — and reported as DEGRADED for exactly this one cycle, deliberately.
	//
	// The tier works again, so "ok" would be true of this instant and would throw away the only
	// evidence that anything happened. R-190's own words: the probe sees the STATE, nothing sees the
	// TRANSITION. A silent self-repair makes a recurring loss undetectable forever, which is strictly
	// worse than the fault it fixes.
	//
	// §8.5 asked whether the hub's existing degraded↔ok edge suffices before building anything new.
	// It does — as a CHANNEL — but only if the agent deliberately reports one degraded cycle: the hub
	// alerts and e-mails on the ok→degraded edge and logs the degraded→ok recovery, so one loss
	// produces exactly one alert pair and the operator learns of it. NOTHING NEW WAS BUILT: no wire
	// change, no hub change, no new event type. The `Feature` text carries the explanation because
	// that is the field the hub puts in the operator's e-mail (the Reason does not travel).
	repair.noteRepaired(targetID, time.Now())
	s = storeGrantRepairedVerdict(targetID, critical)
	repairLogger(repair).Error("store-grant: GRANT WAS MISSING AND HAS BEEN SELF-REPAIRED — investigate the loss (R-190)",
		"target", targetID, "privilege", storeGrantRequiredPriv,
		"action", "felhom-backup-target-apply grant "+targetID, "confirmed_by", "re-read")
	return s
}

// storeGrantHealthyVerdict decides what a HEALTHY probe reports — which is not always "ok".
//
// Split out so the tests exercise this decision rather than a copy of it. An earlier version of this
// guard lived inline and its red-proof PASSED, because the test asserted the latch helper instead of
// the path that consumes it — the same hollow shape this file has now caught twice.
//
// If the tier was repaired inside the report window, the transition is reported even though the grant
// is present: the probe that repaired may have been a log-only one, and without this the host-report
// carries `ok` and the operator never learns the permission vanished (measured live 2026-08-04).
func storeGrantHealthyVerdict(targetID string, critical bool, healthy capability.Status, repairedRecently bool) capability.Status {
	if repairedRecently {
		return storeGrantRepairedVerdict(targetID, critical)
	}
	return healthy
}

// storeGrantRepairedVerdict is the post-repair verdict — the RECORD half of R-190, split out so the
// tests exercise the real thing rather than a copy of it (yesterday's hollow-test lesson).
//
// It reports DEGRADED although the tier now works, and that is the whole point: "ok" would be true of
// this instant and would throw away the only evidence that a permission vanished. The hub raises its
// ok→degraded edge (an operator e-mail) and logs the degraded→ok recovery on the next cycle, so one
// loss produces exactly one alert pair. Nothing new was built for this — no wire change, no hub
// change, no new event type.
//
// The explanation lives in FEATURE because that is the field the hub interpolates into the operator's
// e-mail (`monitor/host_capability.go` emitTransition builds its message from the capability names
// and features; Reason does not travel). Putting it in Reason alone would be a record nobody reads.
func storeGrantRepairedVerdict(targetID string, critical bool) capability.Status {
	return capability.Status{
		Name:     "pve:store-grant:" + targetID,
		Critical: critical,
		Status:   capability.StatusDegraded,
		Feature: "backup tier " + targetID + ": the agent's storage grant was MISSING and has been " +
			"AUTOMATICALLY RESTORED — the tier works now, but a permission that vanished on its own needs investigating (R-190)",
		Reason: "grant absent at probe time; `felhom-backup-target-apply grant " + targetID +
			"` re-applied it and a re-read confirms " + storeGrantRequiredPriv + " is present again",
	}
}

// repairLogger returns the repairer's logger, or the default — the record must survive a nil.
func repairLogger(r *storeGrantRepairer) *slog.Logger {
	if r != nil && r.log != nil {
		return r.log
	}
	return slog.Default()
}

// storeGrantVerdict is the DECISION, split out from the API call so the tests exercise the real
// thing rather than a copy of it. A test that re-implements this branch would pass while production
// diverged — which is the hollow shape this project keeps finding in its own tests.
func storeGrantVerdict(targetID string, critical bool, privs map[string]int, err error) capability.Status {
	s := capability.Status{
		Name:     "pve:store-grant:" + targetID,
		Feature:  "backup tier " + targetID + " readable by the agent (archive listing, restore-test candidacy)",
		Critical: critical,
		Status:   capability.StatusOK,
	}
	if err != nil {
		// Unreachable PVE is UNKNOWN, and unknown is reported as degraded rather than ok: a
		// self-check that fails open converts "I do not know" into "fine".
		s.Status, s.Reason = capability.StatusDegraded, "could not read own permissions: "+err.Error()
		return s
	}
	if privs[storeGrantRequiredPriv] != 1 {
		// Name the storage AND the missing role: "a storage grant is missing" without saying which
		// one costs a diagnosis at 07:00.
		s.Status, s.Reason = capability.StatusDegraded,
			"the agent token lacks "+storeGrantRequiredPriv+" on /storage/"+targetID+
				" (grant FelhomAgentStore there) — this tier's archives are INVISIBLE to the agent and it is never restore-tested"
	}
	return s
}

// logCapabilities logs the privileged-capability self-check at startup: one INFO summary, plus an
// ERROR per degraded capability naming the gated feature (so a missing grant is loud at cutover,
// not days later). Inactive (config-gated off, plumbing healthy — v0.86.0) is counted in the
// summary but never error-logged: disabled ≠ broken. It never exits — serve-degraded.
func logCapabilities(statuses []capability.Status, logger *slog.Logger) {
	ok, total, degraded := capability.Summarize(statuses)
	logger.Info("capabilities self-check",
		"ok", ok, "total", total, "degraded", len(degraded), "inactive", total-ok-len(degraded))
	for _, d := range degraded {
		logger.Error("capability DEGRADED — privileged grant missing (feature impaired until fixed)",
			"capability", d.Name, "feature", d.Feature, "reason", d.Reason, "critical", d.Critical)
	}
}

// runDaemon is the default mode: collect a host-report and POST it to the hub on a
// loop. Requires both proxmox (to collect) and hub config.
func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "daemon: proxmox not configured:", err)
		return 2
	}
	if err := cfg.Hub.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "daemon: hub not configured:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: proxmox client:", err)
		return 1
	}
	client, err := hub.NewClient(cfg.Hub, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: hub client:", err)
		return 1
	}
	hcfg := cfg.Hub.WithDefaults()
	// Storage observer (slice 5): builds the report's storage_targets from Proxmox +
	// non-privileged host reads, enriched with the privileged SMART/lvs reads via HostOps
	// (Phase B). Wired into the collector via the StorageObserver seam (so hub does not
	// import storage).
	hostReader := storage.NewProcHostReader()
	hostOps := newHostOps(cfg, logger)
	observer := storage.NewObserver(px, hostReader, hostOps, logger)
	// Backup + restore-test state store (slice 6): holds the latest backup-per-target +
	// latest restore-test result; the collector reads it via the BackupReporter /
	// RestoreTestReporter seams; the cadence scheduler writes it.
	backupStore := backup.NewStore()
	// PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes the shared
	// store; the collector reads it via the PBSReporter seam. DR-recipe completion (v0.39.0): the
	// collector now reads through a LiveSnapshotReporter that lists snapshots LIVE each collect
	// (cheap GET, last-known-good fallback) so the host-report's pbs coord is present whenever PBS is
	// reachable — even seconds after a restart, before the 6 h verify loop has run. The verify loop
	// keeps Recording into the SAME store (shared last-known-good); targets are resolved once and
	// shared by both. pbsTargets is hoisted so the reporter and the verify loop use one closure.
	pbsStore := pbs.NewSnapshotStore()
	pbsTargets := pbsTargetsFromPVE(cfg, px, logger)
	pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargets, pbsStore, pbs.DefaultLiveSnapshotTimeout, logger)
	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsReporter, cfg.Hub.HostID, version, logger)
	collector.SetBackupTargetResolver(primaryBackupTargetOf(cfg)) // R-109: the recipe names the live target
	// Privileged-capability self-check (v0.44.0): probe the sudoers grants the non-root agent
	// depends on. The probe runs `sudo -n -l` LITERALLY (a policy LIST, never executing the
	// command), so it uses a DIRECT runner regardless of the agent's privileged mode. Probe once at
	// startup (loud on any denial) and attach the snapshot to every hub report; the hub owns the
	// ok→degraded alert. Serve-degraded — a missing grant never blocks startup.
	capProber := capability.Prober{Runner: &proxmox.ExecRunner{Mode: proxmox.RunnerDirect}}
	// DR-tier gate (v0.86.0, DR-tier-by-default): the pbsdr-* capabilities are config-gated — on a
	// box whose DR tier is not configured (no descriptor ever / descriptor disabled) a HEALTHY
	// probe reports "inactive (disabled by configuration)" instead of ok; broken plumbing (binary
	// missing / grant denied) stays DEGRADED regardless (an un-migrated box must never look
	// deliberately off). Late-bound: the pbsdr manager is constructed further down; probes run at
	// report time. nil (pre-assignment) fails ACTIVE — the historical behavior.
	var drConfigured func() bool
	capProber.GateActive = func(gate string) bool {
		if gate == capability.GatePBSDR && drConfigured != nil {
			return drConfigured()
		}
		return true
	}
	// A1 (v0.62.0): compose the PVE pool-read check AROUND the sudo prober (an API read does not
	// belong inside the sudo-policy probe). Non-critical: a degraded pool read means the stale-lock
	// reaper fail-safes (locks stay uncleared) — visible on the hub report, no operator page.
	// R-185: the store-grant probes compose around the sudo prober the same way the pool read does
	// (an API read does not belong inside the sudo-policy probe — the v0.62.0 A1 precedent).
	// R-190: the store-grant probe also REPAIRS a missing grant, through the root wrapper that
	// already exists and is already sudoers-permitted for any storage id — and reports the loss.
	// The runner is the DIRECT one for the same reason the sudo prober uses it: the wrapper is
	// invoked through the privileged path, which prepends sudo itself.
	grantRepairer := &storeGrantRepairer{
		run: (&proxmox.ExecRunner{Mode: proxmox.RunnerMode(cfg.Privileged.Mode)}).Run,
		log: logger,
	}
	probeAll := func(ctx context.Context) []capability.Status {
		out := append(capProber.Probe(ctx), poolReadStatus(ctx, px))
		return append(out, storeGrantStatuses(ctx, px, cfg, grantRepairer)...)
	}
	// (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot
	// already carries the gated view — v0.86.0.)
	collector.SetCapabilityProber(probeAll)
	loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
	interval := time.Duration(hcfg.PollSeconds) * time.Second
	// Heartbeat log-pull (v0.83.0): when the control envelope requests it, the NEXT
	// heartbeat carries the debug ring's tail (newest-kept, byte-capped in the loop).
	if logRing != nil {
		loop.SetLogTailSource(logRing.Lines)
	}

	// Desired-state provider (slice 10A): the hub-served target the reconcile engine converges
	// toward. Starts EMPTY (generation 0) — reconcile is a live no-op until the hub serves intent,
	// exactly like the slice-4 EmptyProvider. The desired.Syncer (wired below) fills it when the
	// control envelope's generation advances. Replaces EmptyProvider in the engine.
	desiredProvider := reconcile.NewCachingProvider()
	// The "Down" channel sync hook: on each heartbeat, fetch desired-state when the generation
	// advances. The loop calls it via the EnvelopeObserver seam (hub does not import desired).
	desiredSyncer := desired.NewSyncer(client, desiredProvider, logger)
	// S5: consume a host_loss restore_directive into an inspectable restore PLAN (derive + surface,
	// execute nothing). The recipe is fetched on-demand (rare directive) via a fresh Collect.
	desiredSyncer.AddConsumer(dr.NewConsumer(func(ctx context.Context) *hub.DRRecipeHostHalf {
		r, err := collector.Collect(ctx)
		if err != nil || r == nil {
			return nil
		}
		return r.DRRecipe
	}, cfg.Backup.RestoreStorage, logger))
	// The signed-jobs runner (slice 10B) is wired as a SECOND envelope observer below (after the
	// gate is built) — when the heartbeat flags pending signed ops, it fetches + verifies + executes.

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	logger.Info("felhom-agent daemon starting",
		"version", version, "host_id", cfg.Hub.HostID, "hub_url", cfg.Hub.URL,
		"interval_s", hcfg.PollSeconds) // hub key intentionally not logged

	// Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue
	// (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving
	// until slice 10), so reconcile is a live no-op: it reads state and computes an
	// empty action set each tick, mutating nothing. The daemon must run cleanly with
	// no desired state and no signers configured — so a journal-open failure is logged
	// and reconcile proceeds journal-less (it has nothing destructive to journal yet).
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err != nil {
			logger.Warn("daemon: cannot ensure journal dir; reconcile runs without a journal", "path", jp, "err", err)
		} else if j, err := reconcile.OpenJournal(jp); err != nil {
			logger.Warn("daemon: cannot open op journal; reconcile runs without a journal", "path", jp, "err", err)
		} else {
			journal = j
			defer journal.Close()
		}
	}
	// The reversibility gate (slice 4 Phase B) sits in front of every mutation. With
	// no signers pinned (the common slice-4 state) the verifier is nil: benign actions
	// pass, destructive intents are refused pending_signature — and reconcile only
	// produces benign actions, so nothing is gated away. A misconfigured signer key is
	// a security misconfig and is fatal.
	verifier, nonceStore, err := buildVerifier(cfg, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "daemon: authz verifier:", err)
		return 2
	}
	if nonceStore != nil {
		defer nonceStore.Close()
	}
	gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)

	// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
	// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
	// known mount-backed target's device returning unmounted, dispatches a benign re-mount
	// (routed through the gate as benign, then HostOps). With no removable/network storage
	// it finds nothing to flag. The re-mount dispatch is off the poll path (a goroutine).
	storageTrigger := make(chan struct{}, 1)
	loop.SetTrigger(storageTrigger)
	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
	// Drive intent store (slice 10 P3 self-heal): persisted, durable-id-keyed enroll/eject/decommission
	// state. Gates the watchdog's self-heal re-mount to ENROLLED drives, and the local API records
	// enroll/eject into it. Open failure → ungated legacy remount (logged) so the daemon still runs.
	var intentReader storage.IntentReader
	intentStateDir := "/var/lib/felhom-agent"
	if cfg.Authz.NonceStorePath != "" {
		intentStateDir = filepath.Dir(cfg.Authz.NonceStorePath) // share the agent's durable state dir
	}
	intentStore, ierr := storage.OpenIntentStore(filepath.Join(intentStateDir, "drive-intents.json"))
	if ierr != nil {
		logger.Warn("storage: intent store unavailable — self-heal runs UNGATED (legacy remount-any)", "err", ierr)
		intentStore = nil
	} else {
		intentReader = intentStore
	}
	// F9: per-guest enrolled-bind record, replayed by ReassertGuestBinds at startup to restore a
	// guest data-drive bind that a re-provision dropped (durable-id-keyed; absent/swapped drives skipped).
	guestBindStore, gbErr := localapi.OpenGuestBindStore(filepath.Join(intentStateDir, "guest-binds.json"))
	if gbErr != nil {
		logger.Warn("storage: guest-bind store unavailable — startup bind re-assert disabled (F9)", "err", gbErr)
		guestBindStore = nil
	}
	// F20-BUG3: persisted disk-format job — lets mkfs run detached from the request and survive an agent
	// restart (RecoverFormatJob re-runs an interrupted durable-id-bound format).
	formatJobStore, fjErr := localapi.OpenFormatJobStore(filepath.Join(intentStateDir, "format-job.json"))
	if fjErr != nil {
		logger.Warn("storage: format-job store unavailable — format restart-recovery disabled (F20-BUG3)", "err", fjErr)
		formatJobStore = nil
	}
	// Impl-2a: source the watchdog's known-DRIVE set from the intent registry + Felhom .mount units
	// (NOT PVE storages via Observe) — a registry-only drive (no PVE dir-storage) is now health-tracked.
	// Real PVE storages (local/local-lvm/pbs) stay Observe-sourced for /disks + reports (observer below).
	driveUnitDir := cfg.Privileged.UnitDir
	if driveUnitDir == "" {
		driveUnitDir = "/etc/systemd/system"
	}
	driveKnown := storage.NewRegistryKnownTargets(driveUnitDir, intentReader, logger)
	// One-time migration: record pre-Impl-2a drives (mounted, with a Felhom unit) as enrolled so the
	// registry-sourced Known() tracks them without their legacy PVE dir-storage. Idempotent.
	if mounts, merr := hostReader.Mounts(); merr == nil {
		storage.ReconcileExistingDrives(driveUnitDir, mounts, intentStore, logger)
	} else {
		logger.Warn("storage: existing-drive migration skipped — mount read failed", "err", merr)
	}
	watchdog := storage.NewWatchdog(storage.WatchdogOptions{
		Targets:   storage.NewCachingKnownTargets(driveKnown, cfg.Storage.KnownRefresh()),
		Liveness:  storage.NewHostLiveness(hostReader, 0),
		Remounter: remounter,
		Intent:    intentReader, // P3: only re-mount enrolled drives
		OnAbsent: func(durableID string) { // P3: clear `ejected` on physical absence (replug rule)
			if intentStore != nil {
				_ = intentStore.OnAbsent(durableID)
			}
		},
		Trigger: func() {
			select {
			case storageTrigger <- struct{}{}:
			default:
			}
		},
		Interval: cfg.Storage.WatchdogInterval(),
		Debounce: cfg.Storage.WatchdogDebounce(),
		Logger:   logger,
	})

	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API:      px,
		Queue:    queue,
		Journal:  journal,
		Provider: desiredProvider, // slice 10A: hub-served desired-state (empty until the hub serves intent)
		Gate:     gate,
		HostID:   cfg.Hub.HostID,
		Logger:   logger,
	})

	// Crash recovery (doc 03 §10): resolve any op that was in flight when the agent
	// last died BEFORE issuing new mutations. For slice 6 this is load-bearing — a
	// restore-test scratch guest leaked by a mid-test crash is torn down here.
	engine.Recover(ctx)

	// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
	// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
	// OR the scratch band / restore storage is misconfigured — the daemon still runs.
	// R-85: persisted per-tier restore-test state + the host-wide one-heavy-op gate, both shared
	// with the local API so a backup and a restore-test can never run together.
	rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
	heavyOps := &backup.InFlight{}
	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, logger)
	// R-189: the host report's restore_tests[] must survive an agent restart. The in-memory store
	// holds only this process's latest run, and under per-archive due-ness the agent will not
	// re-test an archive it has already proven — so without this the hub can report a tier unproven
	// for a whole archive generation after a deploy. Observed live on 2026-08-03: a passing 14.5 GB
	// offsite restore-test reached no host-report at all.
	collector.SetProvenRestoreTests(rtState)

	// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
	// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
	// verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs
	// storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing.
	pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
		Targets: pbsTargets, // the same closure the live reporter uses (shared store, one resolver)
		Store:   pbsStore,
		Cadence: cfg.Backup.PBSVerifyCadence(),
		Logger:  logger,
	})

	// Local API server (slice 8A, doc 03 §6): the per-guest authorization gate the in-guest
	// controller calls over the bridge. Optional — runs only when local_api.enable + listen_addr
	// are configured; a token-store or cert failure disables it WITHOUT killing the daemon (the
	// host still reports/reconciles). The leaf is generated+persisted once so its pin is stable.
	localServers := 0
	var localTokens *localapi.TokenStore
	var intentRec localapi.IntentRecorder
	if intentStore != nil { // avoid a typed-nil interface (would defeat the nil check)
		intentRec = intentStore
	}

	// Signed-jobs runner (slice 10B + P3): the consumer of the hub's signed-jobs queue. On a
	// heartbeat that flags pending signed ops, it fetches each opaque blob, runs it through the gate
	// (the LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time →
	// durable nonce-burn) and, only on all-pass, hands the verified op to the executor chain:
	//   - storage_wipe → re-resolves the DURABLE device id + re-inspects (8C) before mkfs (closes
	//     the 8C data-bearing `pending_signature` gap);
	//   - decommission → records the PERMANENT decommission intent (keyed by the watchdog's storage
	//     durable-id), making the previously-unreachable IntentDecommissioned state reachable ONLY
	//     via a verified operator signature (never customer-confirmable; distinct from a safe eject).
	// With no signers pinned the gate refuses every job (pending_signature) and nothing executes —
	// correct. Wired as a second envelope observer alongside the desired-state syncer. The intent
	// store is opened above (line ~340), so this wiring lives here (after it) rather than earlier.
	wipeExec := signedjobs.NewWipeExecutor(hostOps, logger)
	var decommIntent signedjobs.IntentDecommissioner
	if intentStore != nil { // typed-nil guard (a nil *IntentStore in the interface would pass != nil)
		decommIntent = intentStore
	}
	decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)

	// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
	// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
	// detached restart). Wired as a third chain element. The commit-manager (below) commits a good
	// update after a clean dwell; systemd + the wrapper auto-roll-back a crash-looping one. sudoRunner
	// shells the wrapper verbs via `sudo -n` (same mode the rest of the privileged surface uses).
	suCfg := cfg.SelfUpdate.WithDefaults()
	suMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if suMode == "" {
		suMode = proxmox.RunnerSudo
	}
	suRunner := &proxmox.ExecRunner{Mode: suMode, SudoPath: cfg.Privileged.SudoPath}
	updateExec := selfupdate.NewExecutor(selfupdate.Config{
		URLTemplate: suCfg.URLTemplate,
		Username:    suCfg.Username,
		Token:       suCfg.Token,
		StateDir:    suCfg.StateDir,
		Runner:      suRunner,
		Logger:      logger,
	})
	selfUpdateMgr := selfupdate.NewManager(selfupdate.ManagerConfig{
		StateDir:       suCfg.StateDir,
		RunningVersion: version,
		Dwell:          time.Duration(suCfg.DwellSeconds) * time.Second,
		Runner:         suRunner,
		Logger:         logger,
	})
	collector.SetSelfUpdateReporter(selfUpdateMgr) // heartbeat pending-status field

	// G1: management-plane health observer (break-glass visibility). Read-only + always wired — the
	// dumb watchdog (configs/felhom-mgmt-watchdog.*) does the HEALING agent-independently; this only
	// REPORTS /run/sshd presence + sshd reachability + any recent auto-heal so the hub can surface a
	// recurring clobber before it locks the box out. Port 22 for G1 (H1 passes the felhom-sshd port).
	collector.SetMgmtPlaneReporter(mgmtplane.NewReporter(mgmtplane.DefaultPrivsepDir, mgmtplane.DefaultHealMarker, mgmtplane.DefaultSshdPort))

	jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
	loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))

	// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
	// the pbsdr manager is constructed further down; the closure reads drConfigured at call time
	// (nil until then → preflight reports the tier not applied, which is the honest pre-wire answer).
	escrowCeremonyCfg := &localapi.EscrowCeremonyConfig{
		SudoPath:     cfg.Privileged.SudoPath,
		PBSStorageID: cfg.Escrow.PBSStorageID,
		// Live-reload (v0.89.0): the pbsdr bridge seeds escrow.pbs_storage_id into agent.json on DR
		// convergence; re-read it from disk at preflight time so the row flips green without a
		// restart. Reading the same file the ceremony subprocess loads keeps the preflight honest.
		// On a read error, fall back to the daemon-start snapshot. cfg.SourcePath == "" (all-env
		// config) → nothing to re-read, the snapshot stands.
		CurrentPBSStorageID: func() string {
			if cfg.SourcePath == "" {
				return cfg.Escrow.PBSStorageID
			}
			if c, err := config.Load(cfg.SourcePath); err == nil {
				return c.Escrow.PBSStorageID
			}
			return cfg.Escrow.PBSStorageID
		},
		HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "",
		DRConfigured: func() bool {
			if drConfigured != nil {
				return drConfigured()
			}
			return false
		},
	}
	localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, client, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
	if localTokens != nil {
		defer localTokens.Close()
	}

	// LAN split-horizon resolver: the agent manages a host-side dnsmasq answering *.<customer-domain>
	// with each guest's live LAN IP (so LAN clients reach the box directly — same hostname, same real
	// wildcard cert — instead of hairpinning through Cloudflare). Optional; runs only when
	// lan_resolver.enable is set and a host LAN IP is known (explicit or derived from local_api).
	lanServers := 0
	var lanLoop *lanresolver.Loop
	{
		lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
		if lr.Enabled() && lr.HostIP != "" {
			lrMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if lrMode == "" {
				lrMode = proxmox.RunnerSudo
			}
			lrRunner := &proxmox.ExecRunner{Mode: lrMode, SudoPath: cfg.Privileged.SudoPath}
			mgr := lanresolver.NewManager(lrRunner, lr.HostIP, lr.Upstreams, logger)
			lanLoop = lanresolver.NewLoop(mgr, lr.StateDir, time.Duration(lr.IntervalSeconds)*time.Second, logger)
			logger.Info("lanresolver: enabled", "host_ip", lr.HostIP, "upstreams", lr.Upstreams, "interval_s", lr.IntervalSeconds)
		} else if lr.Enabled() {
			logger.Warn("lanresolver: enabled but no host IP (set lan_resolver.host_ip or local_api.listen_addr) — disabled")
		}
	}

	// Offsite WG tunnel (S3, doc 06 §3.3): keygen + one-shot hub registration + wg-quick@wg-felhom
	// managed from the hub-served desired-state block. DEFAULT OFF (the safety gate): a v0.64.0
	// rollout to a box without explicit wg_tunnel.enabled=true is a no-op — no keygen, no
	// registration, no report stanza.
	wgServers := 0
	var wgLoop *wgtunnel.Loop
	var wgMgr *wgtunnel.Manager // hoisted for the fast-tick convergence source (nil when tunnel disabled)
	var pokeListener *poke.Listener
	{
		wt := cfg.WGTunnel.WithDefaults()
		if wt.Enabled {
			wtMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if wtMode == "" {
				wtMode = proxmox.RunnerSudo
			}
			wtRunner := &proxmox.ExecRunner{Mode: wtMode, SudoPath: cfg.Privileged.SudoPath}
			wgMgr = wgtunnel.NewManager(wtRunner, client, wt.StateDir, logger)
			wgMgr.SetStaleAfter(time.Duration(wt.StaleAfterSeconds) * time.Second)
			wgLoop = wgtunnel.NewLoop(wgMgr, time.Duration(wt.IntervalSeconds)*time.Second, logger)
			desiredSyncer.AddConsumer(wgLoop)      // raw desired-state → the wireguard block
			collector.SetWireguardReporter(wgLoop) // heartbeat status stanza
			logger.Info("wgtunnel: enabled", "interval_s", wt.IntervalSeconds, "state_dir", wt.StateDir)

			// Agent-plane immediate-sync LISTENER (Direction-2a, v0.89.0). Binds a contentless UDP
			// poke socket EXCLUSIVELY to the box's WG /32 (from registered.json) and nudges the hub
			// control loop's out-of-band report trigger — the SAME channel the storage watchdog
			// uses (fan-in; the loop coalesces). Enabled whenever the tunnel is (WG is the only path
			// a poke can arrive on); a lost poke is harmless — the 15-min cycle still reconciles.
			pokeListener = poke.NewListener(
				func() (netip.Addr, bool) { return wgtunnel.LoadAssignedAddr(wt.StateDir) },
				func() {
					select {
					case storageTrigger <- struct{}{}:
					default:
					}
				},
				poke.Port, logger)
			logger.Info("poke: agent-plane sync listener enabled", "port", poke.Port)
		}
	}

	// OOB operator access (H1): the dedicated felhom-sshd instance + port-adaptive belt + health.
	// DEFAULT OFF (oob.enabled) until the operator endpoint + static belt table exist. Consumes the
	// SAME wireguard desired-state block (oob_peer_ip → belt, oob_operator_ssh_key → authorized_keys).
	oobServers := 0
	var oobLoop *felhomsshd.Loop
	{
		oc := cfg.OOB.WithDefaults()
		if oc.Enabled {
			oobMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if oobMode == "" {
				oobMode = proxmox.RunnerSudo
			}
			oobRunner := &proxmox.ExecRunner{Mode: oobMode, SudoPath: cfg.Privileged.SudoPath}
			oobMgr := felhomsshd.NewManager(oobRunner, oc.StateDir, logger)
			oobBelt := felhomsshd.NewBelt(oobRunner, logger)
			oobLoop = felhomsshd.NewLoop(oobMgr, oobBelt, time.Duration(oc.IntervalSeconds)*time.Second, logger)
			desiredSyncer.AddConsumer(oobLoop) // raw desired-state → oob_peer_ip + operator key
			collector.SetOOBReporter(oobLoop)  // heartbeat oob health stanza
			logger.Info("felhomsshd (OOB): enabled", "interval_s", oc.IntervalSeconds, "state_dir", oc.StateDir)
		}
	}

	// PBS DR tier apply-bridge (slice 2): hub-driven — runs whenever the desired-state carries a
	// pbs_dr descriptor (slice 1). No config gate: an absent/disabled descriptor is a no-op, so a
	// v0.80.0 rollout changes nothing until the operator enables the tier on the hub. Adoption
	// first (existing healthy entry → grant + mark, NO consume), verify-pin-before-consume,
	// set-only re-apply, consumed-but-failed is loud (see internal/pbsdr).
	var pbsdrLoop *pbsdr.Loop
	{
		pdMode := proxmox.RunnerMode(cfg.Privileged.Mode)
		if pdMode == "" {
			pdMode = proxmox.RunnerSudo
		}
		pdRunner := &proxmox.ExecRunner{Mode: pdMode, SudoPath: cfg.Privileged.SudoPath}
		secretDir := filepath.Dir(cfg.Backup.PBSSecretPath("x"))
		pdMgr := pbsdr.NewManager(pdRunner, px, client, cfg.WGTunnel.WithDefaults().StateDir,
			secretDir, cfg.SourcePath, logger)
		pbsdrLoop = pbsdr.NewLoop(pdMgr, 60*time.Second, logger)
		desiredSyncer.AddConsumer(pbsdrLoop) // raw desired-state → the pbs_dr block
		collector.SetPBSDRReporter(pbsdrLoop)
		// R-39 leg (c): route the live reporter's per-storage credential probe into the DR bridge, so
		// a 401 becomes a LOUD `auth_failed` the hub self-heals instead of a Warn-and-skip. Without
		// this wiring the probe seam exists but nothing consumes it — and the reporter deliberately
		// skips probing when no sink is attached, so the whole leg would be silently inert.
		pbsReporter.SetAuthSink(pdMgr)
		// Capability gate wiring (v0.86.0): the prober's GatePBSDR now answers from the bridge
		// (descriptor state, marker-backed across restarts) — see the capProber block above.
		drConfigured = pdMgr.DRConfigured
		logger.Info("pbsdr: bridge enabled (hub-driven; no-op until a pbs_dr descriptor arrives)")
	}
	// Startup capability self-check — after the pbsdr gate wiring so the logged snapshot matches
	// what the first report will carry (inactive vs degraded is already resolved here).
	logCapabilities(probeAll(context.Background()), logger)

	// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
	// verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop
	// concurrently; any one returning ends the daemon (ctx cancel tears down the rest).
	errc := make(chan error, 8)
	go func() { errc <- engine.Run(ctx, interval) }()
	go func() { errc <- loop.Run(ctx) }()
	go func() { errc <- watchdog.Run(ctx) }()
	go func() { errc <- scheduler.Run(ctx) }()
	go func() { errc <- pbsLoop.Run(ctx) }()
	go func() { errc <- pbsdrLoop.Run(ctx) }()
	if pokeListener != nil {
		go func() { errc <- pokeListener.Run(ctx) }() // agent-plane immediate-sync listener (v0.89.0)
	}

	// Fast-tick (v0.90.0, R-28): the agent-plane immediacy SECONDARY. While ANY desired-state item is
	// unapplied — most importantly the pre-tunnel window a hub poke cannot reach — pulse the SAME
	// out-of-band trigger the watchdog/poke use, every 30 s, and self-disarm the instant everything
	// converges. Every source is a cached read (no exec/network per tick). The LOUD pbsdr states
	// (consumed_failed/verify_failed) and the destructive pending_signature drift are deliberately
	// EXCLUDED (§8) so a stuck-loud box never hammers.
	fastTick := fasttick.New(storageTrigger, fasttick.DefaultInterval, logger,
		fasttick.SourceFunc(func() (bool, string) {
			if desiredProvider.Generation() == 0 {
				return true, "desired-state not yet fetched"
			}
			return false, ""
		}),
		fasttick.SourceFunc(func() (bool, string) {
			if res, ok := engine.LastResult(); ok && res.Planned-res.Pending > 0 {
				return true, "reconcile drift not yet applied"
			}
			return false, ""
		}),
		fasttick.SourceFunc(func() (bool, string) {
			if pbsdrLoop != nil {
				if st := pbsdrLoop.PBSDRStatus(ctx); st != nil && st.State == "waiting_secret" {
					return true, "pbsdr waiting for its consume-once secret"
				}
			}
			return false, ""
		}),
		fasttick.SourceFunc(func() (bool, string) {
			if wgMgr != nil {
				if desired, operational := wgMgr.TunnelConvergence(); desired && !operational {
					return true, "wireguard tunnel not yet operational"
				}
			}
			return false, ""
		}),
	)
	go func() { errc <- fastTick.Run(ctx) }()
	if localSrv != nil {
		localServers = 1
		// Host-reboot remount fix: BEFORE binding into the guest, re-assert enrolled drive MOUNTS on the
		// host. A `disabled` mount unit (left so by a prior detach) doesn't auto-mount at boot, and kernel
		// re-enumeration can move the device (/dev/sdb→sdc); ReassertEnrolledMounts re-resolves each by
		// filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op.
		// Type-asserted (the concrete op exposes it; the interface stays lean).
		mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts
		if mountReasserter != nil {
			mountReasserter.ReassertEnrolledMounts(ctx)
		}
		// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
		// bind that a re-provision dropped — before serving, so the drive is back in the guest config
		// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
		localSrv.ReassertGuestBinds(ctx)
		// F12 (CAMPAIGN-3, CRITICAL): before re-arming, migrate any already-installed network-storage
		// units to the current template — the units installed before 0.85 carry the network-online
		// ordering cycle that makes each host boot a coin flip (networking lost on one boot, the automount
		// on the next). A general template-drift reconcile (content-hash compare, batched daemon-reload),
		// idempotent. Runs BEFORE the reassert sweep so the re-armed triggers are the fixed units.
		if mountReasserter != nil {
			if n := mountReasserter.MigrateNetworkUnits(ctx); n > 0 {
				logger.Info("network-storage units migrated to the current template (F12 boot-ordering fix)", "migrated", n)
			}
		}
		// RCA fix 1 (AUDIT-nas-cwa-rca-2026-07-11): re-arm idle NAS automount triggers ONCE at startup —
		// covers the host-boot ordering where guests autostarted before the agent (their fresh namespaces
		// missed the triggers). Deliberately NOT in the 20 s ticker: an idle trigger is healthy and must
		// not be churned; guest starts are covered by the guest-hook post-start leg.
		localSrv.ReassertNetworkMounts(ctx)
		// Intermediary-mount GUEST-REBOOT self-heal: re-run the reconcile periodically. A guest reboot
		// (without an agent restart) leaves enrolled drives bound on the HOST but INVISIBLE in the fresh
		// guest namespace (a non-recursive parent bind doesn't carry pre-existing submounts); the periodic
		// reconcile detects the guest can't see the bind and re-fires propagation (force re-bind). Cheap
		// when nothing changed (guest-visibility checks only). 20s ≪ the controller gate's 30s tick.
		go func() {
			t := time.NewTicker(20 * time.Second)
			defer t.Stop()
			for {
				select {
				case <-ctx.Done():
					return
				case <-t.C:
					// Re-assert mounts first (handles a USB that enumerated late after a host reboot, or a
					// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
					if mountReasserter != nil {
						mountReasserter.ReassertEnrolledMounts(ctx)
					}
					localSrv.ReassertGuestBinds(ctx)
				}
			}
		}()
		// CAMPAIGN-3 Part 6: node self-heal watchdog (appliance-gated). One heal — host networking
		// (F12-class defense in depth: any boot leaving networking down is detected + remedied with the
		// exact `systemctl start networking.service` the morning recovery ran by hand). On a byo host the
		// check WARNs but the remedy is structurally unreachable (the Manager gates before any exec).
		{
			shMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if shMode == "" {
				shMode = proxmox.RunnerSudo
			}
			shRunner := &proxmox.ExecRunner{Mode: shMode, SudoPath: cfg.Privileged.SudoPath}
			systemctlBin := cfg.Privileged.Systemctl
			if systemctlBin == "" {
				systemctlBin = "/usr/bin/systemctl"
			}
			shMgr := &selfheal.Manager{
				Appliance: cfg.IsAppliance(),
				Interval:  60 * time.Second,
				Heals: []selfheal.Heal{&selfheal.NetworkingHeal{
					IsActive: selfheal.SystemctlIsActive("networking.service"),
					HasRoute: selfheal.HasDefaultRoute,
					Start: func(ctx context.Context) error {
						_, stderr, err := shRunner.Run(ctx, systemctlBin, "start", "networking.service")
						if err != nil {
							return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(stderr)))
						}
						return nil
					},
					Sleep:  selfheal.RealSleep,
					Logger: logger,
				}},
				Logger: logger,
			}
			mode := "byo"
			if cfg.IsAppliance() {
				mode = "appliance"
			}
			logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
			go shMgr.Watch(ctx)
		}
		// R-54: guest-network watchdog. Closes the OPEN RISK left by
		// INCIDENT-guest-dhclient-killed-2026-07-20 §5 — the guest's DHCP client is started once by
		// ifupdown at boot and nothing supervises it, so its death takes the box off the internet
		// ~1-2 h later, when the lease expires. Host-tier by necessity: a guest with no default
		// route cannot repair its own default route. Deliberately NOT part of the errc fan-out — a
		// watchdog over customer guests must never be able to bring the agent down.
		if gn := cfg.GuestNet.WithDefaults(); gn.Enabled() {
			gnMode := proxmox.RunnerMode(cfg.Privileged.Mode)
			if gnMode == "" {
				gnMode = proxmox.RunnerSudo
			}
			gnRunner := &proxmox.ExecRunner{Mode: gnMode, SudoPath: cfg.Privileged.SudoPath}
			// The guest source is the POOL-VERIFIED one (ListLXC ∩ felhom pool, audit A1) — never a
			// bare ListLXC, which under a broad token would run dhclient inside a co-tenant's guest.
			gnGuests := localapi.NewStaleLockController(px, gnRunner, reconcile.DefaultPool, logger)
			gnWatchdog := guestnet.New(gnRunner, gnGuests, time.Duration(gn.IntervalSeconds)*time.Second, logger)
			gnWatchdog.SetDampers(
				time.Duration(gn.MinHealIntervalSeconds)*time.Second,
				gn.MaxHealsPerHour,
				time.Duration(gn.SettleSeconds)*time.Second,
			)
			collector.SetGuestNetReporter(gnWatchdog)
			go gnWatchdog.Watch(ctx)
		} else {
			logger.Info("guestnet: watchdog disabled by config (guest_net.disable)")
		}
		// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
		localSrv.RecoverFormatJob(ctx)
		// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete
		// the dangling snapshot → start iff onboot). Runs BEFORE the backup loop starts, so a present
		// backup lock is stale by definition (guarded by a no-vzdump-running check; fail-safe otherwise).
		localSrv.RecoverStaleLockedGuests(ctx)
		// F-REBOOT: the startup recovery above only covers a guest left LOCKED by an interrupted
		// backup. A guest that simply ends up stopped-and-unlocked (a `pct reboot` whose shutdown
		// half completed and whose start half never fired — Campaign 8 fault 11, 9m47s of total
		// appliance outage with nothing retrying) needs a PERIODIC check. onboot is the "should be
		// running" signal, so a deliberately stopped guest is never touched.
		go localSrv.WatchGuestPower(ctx)
		go func() { errc <- localSrv.Run(ctx) }()
	}
	if lanLoop != nil {
		lanServers = 1
		go func() { errc <- lanLoop.Run(ctx) }()
	}
	if wgLoop != nil {
		wgServers = 1
		go func() { errc <- wgLoop.Run(ctx) }()
	}
	if oobLoop != nil {
		oobServers = 1
		go func() { errc <- oobLoop.Run(ctx) }()
	}

	// TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS
	// version), commit it after a clean dwell — but only now that core init is done (config parsed,
	// hub loop + storage watchdog + local API all started above). Runs in its own goroutine so the
	// dwell never blocks the daemon; it is NOT one of the errc siblings (it returns after commit and
	// must not end the daemon). A crash before the commit → systemd + the wrapper roll back to .prev.
	go selfUpdateMgr.MaybeCommit(ctx)

	err = <-errc
	stop()                                                                // tear down the siblings on the first exit
	for i := 0; i < 4+localServers+lanServers+wgServers+oobServers; i++ { // wait for the other goroutines
		<-errc
	}
	if err != nil && err != context.Canceled {
		logger.Error("daemon: exited with error", "err", err)
		return 1
	}
	return 0
}

// pbsSecretReader is the seam that reads a PBS storage's token secret (R-39 leg b). Overridable in
// tests; production is readPBSSecretViaWrapper.
var pbsSecretReader = readPBSSecretViaWrapper

// readPBSSecret reads a storage's token secret, preferring a directly-readable file and falling back
// to the root wrapper.
//
// WHY THE FALLBACK EXISTS. The agent runs NON-ROOT and writes /etc/pve/priv/storage/<id>.pw through
// the root wrapper — but /etc/pve/priv is 0700 root:www-data, so it could never read that file back.
// The old code called readTrimmed on it directly, got "permission denied" every cycle, logged a Warn
// and SKIPPED the datastore. That is why the 401 in R-39 went unnoticed for weeks: the one loop that
// could have caught it was blind by construction, not by accident.
//
// The direct read is kept first because a box configured with its own agent-owned secret dir
// (place_copies puts a 0600 felhom-agent copy there) needs no sudo at all; the wrapper is the path
// for the default PRIVDIR case.
func readPBSSecret(ctx context.Context, cfg config.Config, storageID string) (string, error) {
	path := cfg.Backup.PBSSecretPath(storageID)
	if secret, err := readTrimmed(path); err == nil && secret != "" {
		return secret, nil
	}
	return pbsSecretReader(ctx, cfg, storageID)
}

// readPBSSecretViaWrapper shells the root wrapper's `read` verb. The secret arrives on STDOUT and is
// never passed through argv (sudo logs argv) and never logged.
func readPBSSecretViaWrapper(ctx context.Context, cfg config.Config, storageID string) (string, error) {
	dir := filepath.Dir(cfg.Backup.PBSSecretPath(storageID))
	cmd := exec.CommandContext(ctx, "sudo", "-n", pbsdr.WrapperPath, "read", storageID, dir)
	var out, errb bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &errb
	if err := cmd.Run(); err != nil {
		// The wrapper's refusal text is safe to surface (it never echoes the value).
		return "", fmt.Errorf("wrapper read %s: %w: %s", storageID, err, strings.TrimSpace(errb.String()))
	}
	secret := strings.TrimSpace(out.String())
	if secret == "" {
		return "", fmt.Errorf("wrapper read %s: empty secret", storageID)
	}
	return secret, nil
}

// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs
// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each
// (token id from the storage `username`, secret read from <PBSSecretDir>/<id>.pw). A storage
// whose secret/client can't be built is skipped with a warning (the loop still verifies the
// rest). The secret is read at runtime, never logged.
func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets {
	return func(ctx context.Context) ([]pbs.Target, error) {
		stores, err := px.ListStorage(ctx)
		if err != nil {
			return nil, err
		}
		var targets []pbs.Target
		for _, s := range stores {
			if s.Type != "pbs" {
				continue
			}
			secret, err := readPBSSecret(ctx, cfg, s.Storage)
			if err != nil {
				logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
				continue
			}
			c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret, Namespace: s.Namespace})
			if err != nil {
				logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
				continue
			}
			targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c, StorageID: s.Storage})
		}
		return targets, nil
	}
}

// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
// archiveStorageID returns the storage a volid lives on — "felhom-pbs" from
// "felhom-pbs:backup/ct/9201/2026-07-26T12:21:48Z". Empty when there is no storage prefix.
func archiveStorageID(volid string) string {
	if i := strings.Index(volid, ":"); i > 0 {
		return volid[:i]
	}
	return ""
}

// restoreTierForArchive derives the restore tier from THE ARCHIVE'S OWN STORAGE, falling back to
// the configured default target only when the volid carries no storage prefix.
//
// R-82 (found live 2026-07-26): this used to read the tier from cfg.Backup.BackupTarget(), i.e. the
// PRIMARY tier's target. Restoring a `felhom-pbs:` archive on a box whose primary is "local" was
// therefore classified "local" and got the 10-MINUTE local wait instead of the generous PBS one —
// the wait expired mid-restore at 600s, teardown fired against a still-restoring guest, and the
// scratch leaked. Exactly the failure RestoreTestSpec.RestoreTaskTimeout's doc comment predicts.
//
// The tier-aware machinery was already correct; it was fed the wrong input. With more than one tier
// configured, "the configured target" is no longer a proxy for "the tier this archive belongs to".
func restoreTierForArchive(ctx context.Context, px *proxmox.Client, archive, fallbackTarget string) string {
	id := archiveStorageID(archive)
	if id == "" {
		id = fallbackTarget
	}
	return storageTier(ctx, px, id)
}

func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
	stores, err := px.ListStorage(ctx)
	if err != nil {
		return "local"
	}
	for _, s := range stores {
		if s.Storage == storageID && s.Type == "pbs" {
			return "pbs"
		}
	}
	return "local"
}

// restoreTaskTimeout returns the tier-aware restore-task wait: the generous configured PBS timeout
// for a WAN (pbs-tier) restore, else 0 (→ WaitOptions' 10m default) for a local restore. S4.1: a
// too-short wait kills a WAN restore mid-flight → mid-restore teardown → leaked scratch.
func restoreTaskTimeout(cfg config.Config, tier string) time.Duration {
	if tier == "pbs" {
		return cfg.Backup.RestoreTestPBSRestoreTimeout()
	}
	return 0
}

// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
func readTrimmed(path string) (string, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	s := strings.TrimSpace(string(b))
	if s == "" {
		return "", fmt.Errorf("empty file %s", path)
	}
	return s, nil
}

// primaryBackupTargetOf returns the resolver the DR recipe uses to name WHICH storage holds this box's
// local whole-guest archives (R-109).
//
// It reads the PRIMARY tier out of cfg.Backup.BackupTiers() rather than calling BackupTarget() directly.
// Both return the same string today — BackupTiers() builds tier 0 from BackupTarget() — but the tier
// list is the function the scheduler itself consults, so if primary-tier derivation ever changes the
// recipe follows it instead of quietly disagreeing with the backup. One state, one owner.
//
// cfg is captured BY VALUE on purpose: that is the daemon-start snapshot, which is the config actually
// in effect. See SetBackupTargetResolver for why re-reading agent.json here would be wrong.
func primaryBackupTargetOf(cfg config.Config) func() hub.ConfiguredBackupTarget {
	return func() hub.ConfiguredBackupTarget {
		tiers, _ := cfg.Backup.BackupTiers() // warnings are logged where the tiers are armed
		for _, t := range tiers {
			if t.Primary {
				return hub.ConfiguredBackupTarget{StorageID: t.TargetID, Known: true}
			}
		}
		// Unreachable with today's BackupTiers (tier 0 is always primary), and if that ever stops being
		// true the recipe says "I could not tell" rather than picking a tier at random.
		return hub.ConfiguredBackupTarget{}
	}
}

// buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
// machinery still works on-demand via --selftest=restore-test.
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, logger *slog.Logger) *backup.Scheduler {
	// R-86: this is the EVALUATION interval, not the trigger. What decides a test happens is the
	// per-archive due-check in internal/backup/restoretest_due.go.
	cadence := cfg.Backup.RestoreTestEvalInterval()
	if cadence > 0 {
		if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
			logger.Warn("daemon: restore-test disabled (config invalid)", "err", err)
			cadence = 0
		}
	}
	if cadence > 0 && cfg.Backup.RestoreTestLegacyCadenceInUse() {
		// Said ONCE, at start-up, naming both replacements: a key whose meaning changed under a box
		// without a word is the silent repurposing R-86 §8.3 forbids.
		logger.Warn("daemon: backup.restore_test_cadence_seconds is DEPRECATED — R-86 replaced the interval trigger with a per-archive due-check; this value now seeds the SETTLE lag only. Set backup.restore_test_settle_seconds and backup.restore_test_eval_interval_seconds explicitly",
			"settle", cfg.Backup.RestoreTestSettle(), "eval_interval", cadence)
	}
	min, max := cfg.Backup.ScratchBand()
	target := cfg.Backup.BackupTarget()
	runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
	// Every configured tier is a rotation candidate, not just the primary.
	cfgTiers, _ := cfg.Backup.BackupTiers() // warnings already logged where the tiers are armed
	tierIDs := make([]string, 0, len(cfgTiers))
	for _, t := range cfgTiers {
		tierIDs = append(tierIDs, t.TargetID)
	}
	return backup.NewScheduler(backup.SchedulerOptions{
		Runner: engine,
		Pick:   runner.PickRestoreCandidate,
		Store:  store,
		// R-85 (1.1): the spec is built PER RUN, from the archive that was picked.
		//
		// This used to be an immediately-invoked function, so storageTier() and
		// restoreTaskTimeout() ran ONCE at daemon start and their result was reused for every run
		// forever. That froze the tier — and with it the timeout — making an offsite restore-test
		// impossible to schedule, and leaving any storage-type or config change stale until the
		// daemon restarted.
		//
		// The tier comes from the ARCHIVE (restoreTierForArchive, the v0.100.0 rule), never from
		// the configured target: config-derived was what classified a PBS archive as "local" and
		// killed a 14.46 GB WAN restore at the 10-minute local bound.
		Spec: func(ctx context.Context, archive string) reconcile.RestoreTestSpec {
			tier := restoreTierForArchive(ctx, px, archive, target)
			return reconcile.RestoreTestSpec{
				RestoreStorage:     cfg.Backup.RestoreStorage,
				ScratchMin:         min,
				ScratchMax:         max,
				SourceTier:         tier,
				RestoreTaskTimeout: restoreTaskTimeout(cfg, tier),
			}
		},
		Cadence: cadence,
		// R-86: the settle lag — how long an archive must have sat before it is a candidate. With
		// the per-archive due-check, this plus the archive rhythm is the whole schedule.
		Settle: cfg.Backup.RestoreTestSettle(),
		Logger: logger,

		// R-85: rotate across EVERY configured tier, oldest-proven first (operator ruling, Option 1).
		// Before this the scheduler only ever saw cfg.Backup.BackupTarget(), so the offsite tier's
		// archives were never candidates and the DR tier went unproven for its whole existence.
		// R-86 demoted that ordering to the tie-break BETWEEN DUE TIERS and widened this picker to
		// the settle-aware one, which is what makes due-ness per archive generation.
		Tiers:    tierIDs,
		TierPick: runner.PickSettledRestoreCandidateOn,
		State:    rtState,
		InFlight: inFlight,
	})
}

// buildLocalAPIServer constructs the per-guest local-API server (slice 8A, doc 03 §6) when
// configured. It opens the durable hashed token store and ensures the persisted self-signed
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
// fixed. The opened token store is returned via outTokens so the caller can Close it.
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, inFlight *backup.InFlight, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, hubClient *hub.Client, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
	if !cfg.LocalAPI.Enabled() {
		return nil
	}
	if err := cfg.LocalAPI.Validate(); err != nil {
		logger.Warn("daemon: local-api disabled (config invalid)", "err", err)
		return nil
	}
	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
	if err != nil {
		logger.Warn("daemon: local-api disabled (token store)", "err", err)
		return nil
	}
	*outTokens = tokens
	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
	cert, fp, generated, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
	if err != nil {
		logger.Warn("daemon: local-api disabled (leaf cert)", "err", err)
		return nil
	}
	if generated {
		// B.1: a freshly-minted leaf invalidates every already-issued bootstrap pin. LOUD so an
		// accidental regeneration (e.g. a state-dir move during a reinstall) is caught immediately.
		logger.Warn("local-api leaf REGENERATED — any previously issued bootstrap pins are now INVALID; controllers will fail the pin check until re-bootstrapped (or restore the prior leaf)",
			"fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
	} else {
		logger.Info("local-api leaf LOADED", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
	}
	// v0.48.0: ride the served leaf fp on every host report so the hub can detect a re-key fleet-wide.
	collector.SetLeafFingerprint(fp)
	// R-82: ONE RUNNER PER TIER. The runner holds its target, mode, notes and retention as
	// immutable construction state, and localPruneSpec reads that retention — so parameterising a
	// single runner by target would risk a call pairing tier A's target with tier B's retention.
	// One runner per tier keeps each tier's policy structurally inseparable from its target.
	backupTiers, tierWarnings := cfg.Backup.BackupTiers()
	for _, wmsg := range tierWarnings {
		// LOUD on purpose: a silently dropped backup tier is an "applied and empty" DR tier, which
		// is the exact fault R-82 exists to fix. Never downgrade this to DEBUG.
		logger.Error("backup tier REJECTED — this tier will never run", "detail", wmsg)
	}
	apiTiers := make([]localapi.BackupTier, 0, len(backupTiers))
	var runner *backup.BackupRunner
	for _, t := range backupTiers {
		prune := ""
		if t.KeepLast > 0 {
			prune = fmt.Sprintf("keep-last=%d", t.KeepLast)
		}
		// Pruning a PBS target is allowed ONLY for an additional tier with an explicit keep_last
		// (the primary's target AND retention both default, so it could prune the DR by accident).
		allowPBSPrune := !t.Primary && t.KeepLast > 0
		r := backup.NewBackupRunnerFull(px, t.TargetID, "", "felhom local-api", prune, t.WaitTimeout, allowPBSPrune, logger)
		if t.Primary {
			runner = r
		}
		apiTiers = append(apiTiers, localapi.BackupTier{
			TargetID:    t.TargetID,
			Cadence:     t.Cadence,
			WaitTimeout: t.WaitTimeout,
			Primary:     t.Primary,
			Service:     r,
		})
		logger.Info("backup tier armed", "target", t.TargetID, "cadence", t.Cadence.String(),
			"keep_last", t.KeepLast, "wait_timeout", t.WaitTimeout.String(),
			"prune_pbs_allowed", allowPBSPrune, "primary", t.Primary)
	}
	// Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown
	// (same fenced ExecRunner the host-storage + provision back-half use).
	gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if gaMode == "" {
		gaMode = proxmox.RunnerSudo
	}
	guestBinder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
	// R-199 (v0.125.0) — chain links 6->8, assembled here and ONLY here. The fetcher is this daemon's
	// own hub client (per-host key, self-scoped server-side), so the recoverer can never read another
	// host's blob even if asked to. `client` is the same one the report loop uses; a nil hub config
	// cannot reach this line (the daemon exits above), so the seam is always live in production —
	// which is the point: links 6 and 7 spent months existing without a caller.
	escrowRecoverer := escrow.OffsiteKeyRecoverer{
		Fetch: func(ctx context.Context) ([]byte, bool, error) {
			resp, ferr := hubClient.FetchIdentityEscrow(ctx)
			if ferr != nil {
				return nil, false, ferr
			}
			if !resp.Present || resp.IdentityEscrowB64 == "" {
				return nil, false, nil
			}
			blob, derr := base64.StdEncoding.DecodeString(resp.IdentityEscrowB64)
			if derr != nil {
				return nil, false, fmt.Errorf("hub served a malformed escrow blob (not base64)")
			}
			return blob, true, nil
		},
	}
	srv, err := localapi.NewServer(localapi.Options{
		EscrowRecovery: escrowRecoverer,
		ListenAddr:    cfg.LocalAPI.ListenAddr,
		Cert:          cert,
		AgentVersion:  version, // v0.82.0: the X-Felhom-Agent-Version capability channel
		Guests:        px,
		Backups:       runner,
		BackupTiers:   apiTiers, // R-82: primary first; untargeted endpoints act on the primary
		InFlight:      inFlight, // R-85: shared with the restore-test scheduler (Scenario F)
		Store:         store,
		Storage:       observer,
		DriveTargets:  driveTargets,                    // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages)
		Smart:         storage.NewSmartReader(hostOps), // v0.95.0 Fix B: SMART for the union-path drives
		HostReader:    storage.NewProcHostReader(),     // Impl-2b: durableIDForMount raw-mount fallback + role gate
		Tokens:        tokens,
		BackupCadence: cfg.Backup.BackupCadence(),
		// Disk management (slice 8C): the privileged host surface + the data-bearing wipe gate.
		Disks:       hostOps,
		DiskGate:    storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID},
		Guests2:     px,
		GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest
		Memory:      px,          // v0.90.0 R-24: guest RAM resize (SetConfig live cgroup apply)
		// Network storage (NAS) — Part A1: the privileged host network-mount surface (NFS/SMB automount).
		NetStorage: hostOps,
		// E-2a: the fenced root shim for the backup-target move. Same runner mode as every other
		// privileged call; the sudoers vector is what actually bounds it.
		Privileged:     &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath},
		ConfigPath:     cfg.SourcePath,
		StateDir:       cfg.WGTunnel.WithDefaults().StateDir,
		SmbCredsDir:    cfg.Privileged.SmbCredsDir,
		ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap
		// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
		// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
		// A1 (v0.62.0): the scan is restricted to felhom-pool members (ownership proven, not assumed).
		StaleLock:  localapi.NewStaleLockController(px, &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, reconcile.DefaultPool, logger),
		Intent:     intent,     // slice 10 P3: record enroll/eject intent for self-heal
		GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert
		FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery

		// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
		// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
		HostMetrics: collector,
		HostID:      cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op
		LogRing:     logRing,        // v0.83.0: GET /debug/logs — the always-DEBUG capture ring
		// Controller-driven escrow ceremony (v0.88.0): the customer wizard's agent half.
		EscrowCeremony: escrowCeremony,
		Logger:         logger,
	})
	if err != nil {
		logger.Warn("daemon: local-api disabled (server build)", "err", err)
		return nil
	}
	return srv
}

// storageGateAdapter bridges the local-API disk-format path to the reversibility gate, TIERED by the
// agent's authoritative device-role verdict: a USER-DATA data-bearing wipe is customer-confirmable
// (durable-id-bound, no signature); a SYSTEM/BACKUP wipe stays operator-signature (unsigned →
// pending_signature; the signed completion runs via the signed-jobs runner).
type storageGateAdapter struct {
	gate   *reconcile.Gate
	hostID string
}

func (a storageGateAdapter) AuthorizeWipe(req localapi.WipeRequest) localapi.WipeDecision {
	dec := a.gate.AuthorizeStorageWipe(reconcile.StorageWipeAuthz{
		HostID:           a.hostID,
		Role:             req.Role,
		DeviceDurableID:  req.DeviceDurableID,
		Confirmed:        req.Confirmed,
		ConfirmDurableID: req.ConfirmDurableID,
	}, nil) // no operator signature on the inline path → system/backup is pending_signature
	tier := "destructive"
	if dec.Disposition == reconcile.CustomerConfirmable {
		tier = "customer_confirmable"
	}
	return localapi.WipeDecision{
		Allowed:           dec.Allowed,
		Tier:              tier,
		Reason:            string(dec.Reason),
		NeedsConfirmation: dec.Reason == reconcile.ReasonPendingConfirmation,
	}
}

// reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
// configured nonce store (both are durable agent state), falling back to the standard
// host state dir when the nonce store is unset.
func reconcileJournalPath(cfg config.Config) string {
	if p := cfg.Authz.NonceStorePath; p != "" {
		return filepath.Join(filepath.Dir(p), "journal.log")
	}
	return "/var/lib/felhom-agent/journal.log"
}

// buildVerifier constructs the operator-signed-op verifier from config. With no signers
// pinned it returns (nil, nil, nil) — the gate then refuses any destructive intent as
// pending_signature (and reconcile serves none). With signers it requires a durable
// nonce-store path (anti-replay must survive restarts) and parses every pinned key —
// a bad key or missing path is a fatal security misconfig.
func buildVerifier(cfg config.Config, logger *slog.Logger) (reconcile.OpVerifier, *authz.FileNonceStore, error) {
	if len(cfg.Authz.Signers) == 0 {
		logger.Info("daemon: no operator signers pinned; destructive ops will be refused pending_signature")
		return nil, nil, nil
	}
	if cfg.Authz.NonceStorePath == "" {
		return nil, nil, fmt.Errorf("authz.nonce_store_path is required when signers are configured")
	}
	signers := make([]authz.AllowedSigner, 0, len(cfg.Authz.Signers))
	for _, s := range cfg.Authz.Signers {
		as, err := authz.NewAllowedSigner(s.KeyID, authz.KeyRole(s.Role), s.PublicKey)
		if err != nil {
			return nil, nil, fmt.Errorf("pinned signer %q: %w", s.KeyID, err)
		}
		signers = append(signers, as)
	}
	store, err := authz.OpenFileNonceStore(cfg.Authz.NonceStorePath)
	if err != nil {
		return nil, nil, fmt.Errorf("nonce store: %w", err)
	}
	logger.Info("daemon: operator signers pinned", "count", len(signers))
	return authz.New(signers, store, cfg.Hub.HostID), store, nil
}

// runSelftestControllerSwap exercises the agentic controller-update swap primitive directly (Phase 1):
// it swaps guest -vmid's controller to -image, verifies it comes up healthy, and ROLLS BACK if not.
// The target image must already be pulled in the guest (the controller pre-pulls it in the real flow).
func runSelftestControllerSwap(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int, image string) int {
	if vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=controller-swap: -vmid is required")
		return 1
	}
	if !localapi.ValidControllerImage(image) {
		fmt.Fprintln(os.Stderr, "selftest=controller-swap: -image must be gitea.dooplex.hu/admin/felhom-controller:<semver>")
		return 1
	}
	gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if gaMode == "" {
		gaMode = proxmox.RunnerSudo
	}
	binder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
	swapper := localapi.NewControllerSwapper(binder, "/var/lib/felhom-agent", logger)

	cur, _ := swapper.CurrentImage(ctx, vmid)
	fmt.Printf("=== felhom-agent %s selftest=controller-swap (vmid=%d) ===\n", version, vmid)
	fmt.Printf("  current image: %s\n  target  image: %s\n", cur, image)

	st := swapper.Swap(ctx, vmid, image)
	fmt.Printf("  --- result ---\n  state=%s current=%s previous=%s\n", st.State, st.Current, st.Previous)
	if st.Error != "" {
		fmt.Printf("  error: %s\n", st.Error)
	}
	if st.State == "done" {
		fmt.Println("=== selftest=controller-swap OK (swapped + healthy) ===")
		return 0
	}
	fmt.Println("=== selftest=controller-swap FAILED (rolled back if previous existed) ===")
	return 1
}

// runSelftestHub validates hub config, does ONE collect + report, and prints the
// report it would send plus the envelope it got back.
func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if err := cfg.Hub.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: hub not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	client, err := hub.NewClient(cfg.Hub, logger)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: hub client:", err)
		return 1
	}
	observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
	// Wire the LIVE PBS reporter here too (v0.39.0): selftest=hub is a separate one-shot process — no
	// verify loop runs — so a nil reporter previously yielded pbs_snapshots:[] and an absent dr_recipe
	// pbs coord. The live reporter lists snapshots directly (fresh store, last-known-good fallback) so
	// the selftest reflects exactly what a freshly-restarted daemon's first collect emits.
	pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargetsFromPVE(cfg, px, logger), pbs.NewSnapshotStore(), pbs.DefaultLiveSnapshotTimeout, logger)
	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, pbsReporter, cfg.Hub.HostID, version, logger)
	// R-109: wire the backup-target resolver here TOO. Without it selftest=hub would print a recipe whose
	// backup_target reads unknown/agent_backup_config_unavailable while the daemon's is resolved — and
	// this one-shot exists precisely so "the report it would send" can be trusted to match.
	collector.SetBackupTargetResolver(primaryBackupTargetOf(cfg))

	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=hub (host_id=%s url=%s) ===\n", version, cfg.Hub.HostID, cfg.Hub.URL)
	report, err := collector.Collect(ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] collect:", err)
		return 1
	}
	if b, e := json.MarshalIndent(report, "  ", "  "); e == nil {
		fmt.Println("  --- report it would send ---")
		fmt.Println("  " + string(b))
	}
	env, err := client.Report(ctx, report)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] report:", err)
		return 1
	}
	if b, e := json.MarshalIndent(env, "  ", "  "); e == nil {
		fmt.Println("  --- control envelope received ---")
		fmt.Println("  " + string(b))
	}
	fmt.Println("=== selftest=hub OK ===")
	return 0
}

// runSelftestStorage is the live storage harness (slice 5 Phase B, for the USB runbook).
// It needs Proxmox config only (NO hub) so it runs standalone on the Proxmox host:
//   - observe pass: print the full StorageTarget table incl. the privileged SMART summary
//     and thin-pool data+metadata fill.
//   - -watch D: run the watchdog verbose for D with the re-mount response LIVE, so the
//     operator can physically cycle a drive and watch detect → report → re-mount.
func runSelftestStorage(ctx context.Context, cfg config.Config, logger *slog.Logger, watch time.Duration) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	hostReader := storage.NewProcHostReader()
	hostOps := newHostOps(cfg, logger)
	observer := storage.NewObserver(px, hostReader, hostOps, logger)

	octx, cancel := context.WithTimeout(ctx, 60*time.Second)
	fmt.Printf("=== felhom-agent %s selftest=storage ===\n", version)
	targets, err := observer.Observe(octx)
	cancel()
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] observe:", err)
		return 1
	}
	fmt.Printf("  observed %d storage target(s):\n", len(targets))
	for _, t := range targets {
		fmt.Printf("   - %-12s type=%-9s state=%-12s reach=%-5v class=%-4s durable=%s\n",
			t.Name, t.Type, t.State, t.Reachable, t.ClassHint, t.DurableID)
		fmt.Printf("       usage %s/%s (%.0f%%)  mount=%q dev=%q\n",
			gib(t.UsedBytes), gib(t.TotalBytes), t.UsedFraction*100, t.MountPath, t.BackingDevice)
		fmt.Printf("       smart: health=%s%s\n", t.Smart.Health, smartCounters(t.Smart))
		if t.ThinPool != nil {
			meta := "n/a"
			if t.ThinPool.MetadataUsedFraction != nil {
				meta = fmt.Sprintf("%.1f%%", *t.ThinPool.MetadataUsedFraction*100)
			}
			fmt.Printf("       thin-pool: data=%.1f%% metadata=%s\n", t.ThinPool.DataUsedFraction*100, meta)
		}
	}

	if watch <= 0 {
		fmt.Println("=== selftest=storage OK (observe pass; pass -watch D for the live watchdog) ===")
		return 0
	}

	// Live watchdog window: re-mount response active. Cycle a drive and watch the logs.
	fmt.Printf("  --- watching for %s (cycle a drive now; detect → report → re-mount) ---\n", watch)
	wctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()
	wctx, cancel2 := context.WithTimeout(wctx, watch)
	defer cancel2()

	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
	wd := storage.NewWatchdog(storage.WatchdogOptions{
		Targets:   storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
		Liveness:  storage.NewHostLiveness(hostReader, 0),
		Remounter: remounter,
		Trigger:   func() { logger.Info("storage: (selftest) would send out-of-band host-report now") },
		Interval:  cfg.Storage.WatchdogInterval(),
		Debounce:  cfg.Storage.WatchdogDebounce(),
		Logger:    logger,
	})
	_ = wd.Run(wctx)
	fmt.Println("=== selftest=storage watch window ended ===")
	return 0
}

// smartCounters renders the non-nil SMART counters compactly for the selftest table.
func smartCounters(s hub.SmartSummary) string {
	var parts []string
	add := func(name string, v *int) {
		if v != nil {
			parts = append(parts, fmt.Sprintf("%s=%d", name, *v))
		}
	}
	add("temp", s.TemperatureC)
	add("poh", s.PowerOnHours)
	add("realloc", s.ReallocatedSectors)
	add("pending", s.PendingSectors)
	add("offline_unc", s.OfflineUncorrectable)
	add("crit_warn", s.CriticalWarning)
	add("media_err", s.MediaErrors)
	add("pct_used", s.PercentageUsed)
	if len(parts) == 0 {
		return ""
	}
	return " " + strings.Join(parts, " ")
}

// runSelftestBackup runs one vzdump of -vmid to the configured local backup target and
// prints the resulting Backup record. Standalone (no hub). Live + benign.
func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if vmid == 0 {
		fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
		return 2
	}
	target := cfg.Backup.BackupTarget()
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, target)
	runner := backup.NewBackupRunner(px, target, "", "felhom selftest", "", logger)
	rec, err := runner.Backup(ctx, vmid)
	printJSON("backup record", rec)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] backup:", err)
		return 1
	}
	fmt.Printf("=== selftest=backup OK (crash-consistent=%v, archive=%s) ===\n", rec.CrashConsistent, rec.Archive)
	return 0
}

// runSelftestRestoreTest runs one self-restore-test (restore → net-link-down → boot → verify
// running → teardown) of -archive (or the newest backup on the local target) into a scratch
// guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
// crashed test is reaped before this run.
// runSelftestRestoreTestDue prints the per-tier DUE verdict the scheduler would act on, and what
// each evaluation COST — read-only, so it is safe on any box at any time.
//
// It exists for two reasons R-86 needed and could not get from a log line. First, the due-check's
// verdict is the whole schedule now: "why did nothing run last night?" is answerable only by asking
// the same question the scheduler asks, against the same storages, in the same order. Second, the
// evaluation interval had to be chosen from a MEASURED cost rather than a guess — an offsite tier's
// candidate lookup crosses the WAN, and a monitoring loop that costs more than it is worth is how a
// check becomes the load. It reuses the daemon's own construction path (buildRestoreTestScheduler),
// so what it prints is what the daemon would decide, not a re-derivation of it.
func runSelftestRestoreTestDue(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
	sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, logger)

	fmt.Printf("eval_interval=%s settle=%s\n", cfg.Backup.RestoreTestEvalInterval(), cfg.Backup.RestoreTestSettle())
	start := time.Now()
	verdicts := sched.EvaluateDue(ctx)
	total := time.Since(start)
	if len(verdicts) == 0 {
		fmt.Println("no tiers configured for restore-testing (or rotation not wired)")
		return 0
	}
	rc := 0
	for _, v := range verdicts {
		proven, _ := rtState.ProvenArchive(v.Target)
		fmt.Printf("tier=%-16s due=%-5v archive=%q landed=%s proven=%q\n  reason: %s\n",
			v.Target, v.Due, v.Archive, formatOrDash(v.Landed), proven, v.Reason)
		if v.Err != nil {
			// A tier we could not list is UNKNOWN, and it is a non-zero exit: an unreadable tier is
			// a real condition, not a quiet "nothing to do".
			fmt.Printf("  ERROR: %v\n", v.Err)
			rc = 3
		}
	}
	// Per-tier timing, measured one tier at a time so the WAN leg is attributable (R-86 Part 1.4).
	for _, v := range verdicts {
		t0 := time.Now()
		_ = sched.EvaluateDueTier(ctx, v.Target)
		fmt.Printf("cost tier=%-16s one_lookup=%s\n", v.Target, time.Since(t0).Round(time.Millisecond))
	}
	fmt.Printf("cost all_tiers=%s\n", total.Round(time.Millisecond))
	return rc
}

// formatOrDash renders a time, or "-" when it is zero (no archive).
func formatOrDash(t time.Time) string {
	if t.IsZero() {
		return "-"
	}
	return t.UTC().Format(time.RFC3339)
}

func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest=restore-test:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	// Build a journaled engine so a leaked scratch from a prior crash is recoverable.
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
	})

	fmt.Printf("=== felhom-agent %s selftest=restore-test ===\n", version)
	fmt.Println("  --- recover: reaping any leaked scratch from a prior crashed test ---")
	rec := engine.Recover(ctx)
	fmt.Printf("  recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)

	target := cfg.Backup.BackupTarget()
	if archive == "" {
		runner := backup.NewBackupRunner(px, target, "", "", "", logger)
		archive, err = runner.PickRestoreCandidate(ctx)
		if err != nil {
			fmt.Fprintln(os.Stderr, "  [FAIL] pick backup:", err)
			return 1
		}
		if archive == "" {
			fmt.Fprintln(os.Stderr, "  [FAIL] no backup available on", target, "(run --selftest=backup first)")
			return 1
		}
	}
	min, max := cfg.Backup.ScratchBand()
	fmt.Printf("  restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
	rtTier := restoreTierForArchive(ctx, px, archive, target)
	res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
		Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
		ScratchMin: min, ScratchMax: max, SourceTier: rtTier,
		RestoreTaskTimeout: restoreTaskTimeout(cfg, rtTier),
	})
	printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
	if res.Skipped {
		fmt.Println("=== selftest=restore-test SKIPPED (no free scratch VMID in band) ===")
		return 0
	}
	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] restore-test (scratch %d): %v\n", res.ScratchVMID, res.Err)
		return 1
	}
	fmt.Printf("=== selftest=restore-test OK (scratch %d restored+booted+verified+torn-down in %s) ===\n", res.ScratchVMID, res.Duration.Round(time.Second))
	return 0
}

// runSelftestBringUp runs the REAL journaled bring-up job (slice 7) on-demand: restore →
// reset identity → size → attach mounts → start link-up, then tears the guest down by default
// (a selftest must not leave a guest running) unless -keep. -mode picks provision (golden, fresh
// identity) or dr (customer backup, preserve continuity). It first Recovers, so a leaked guest
// from a prior crashed bring-up is reaped before this run.
// bringUpSizing carries the OS-rootfs / Docker-data sizing knobs from the CLI into a bring-up.
// Defaults (0/"") keep the golden's baked sizes; the provisioning spec sources these per-customer
// (flags now; the slice-10 hub storage manifest later — see bringup.go GuestMount comment).
type bringUpSizing struct {
	RootfsGrowGB  int
	DataVolGrowGB int
	DataVolMount  string
	SysDataGrowGB int
	SysDataMount  string
	Cores         int // 0 = keep golden default (CPU-core cap applied pre-start)
	MemoryMB      int // 0 = keep golden default (RAM cap in MiB, applied pre-start)
}

func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Logger, mode, archive string, vmid int, hostname string, keep bool, sizing bringUpSizing) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	var bmode reconcile.BringUpMode
	switch mode {
	case "provision":
		bmode = reconcile.ModeProvision
	case "dr":
		bmode = reconcile.ModeDRGuestLoss
	default:
		fmt.Fprintf(os.Stderr, "selftest=bring-up: -mode must be provision|dr (got %q)\n", mode)
		return 2
	}
	if archive == "" || vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=bring-up requires -archive <volid> and -vmid <N>")
		return 2
	}
	if cfg.Backup.RestoreStorage == "" {
		fmt.Fprintln(os.Stderr, "selftest=bring-up requires backup.restore_storage in config")
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	// GL-5: DR bring-up swaps the structural binds (mp8/mp9) via root pct ops — wire the same
	// Runner shape the provision back-half uses. Provision mode never touches it.
	rMode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if rMode == "" {
		rMode = proxmox.RunnerSudo
	}
	hostRunner := &proxmox.ExecRunner{Mode: rMode, SudoPath: cfg.Privileged.SudoPath}
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
		HostRunner: hostRunner,
	})

	fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
	fmt.Println("  --- recover: reaping any half-built guest from a prior crashed bring-up ---")
	rec := engine.Recover(ctx)
	fmt.Printf("  recover: examined=%d bring_up_rolled_back=%d bring_up_clean=%d scratch_destroyed=%d\n",
		rec.Examined, rec.BringUpRolledBack, rec.BringUpClean, rec.ScratchDestroyed)

	spec := reconcile.BringUpSpec{
		Mode: bmode, Archive: archive, VMID: vmid, RestoreStorage: cfg.Backup.RestoreStorage,
		Hostname: hostname, KeepMAC: bmode == reconcile.ModeDRGuestLoss,
		Pool:  reconcile.DefaultPool, // restore into the felhom pool (both provision + DR); required under the scoped token
		Cores: sizing.Cores, MemoryMB: sizing.MemoryMB,
		RootfsGrowGB: sizing.RootfsGrowGB, DataVolGrowGB: sizing.DataVolGrowGB, DataVolMount: sizing.DataVolMount,
		SysDataGrowGB: sizing.SysDataGrowGB, SysDataMount: sizing.SysDataMount,
		IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
	}
	fmt.Printf("  bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
	res := engine.RunBringUp(ctx, spec)
	printJSON("bring-up record", res)

	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] bring-up (vmid %d): %v\n", vmid, res.Err)
		// On failure the job already compensating-rolled-back; nothing to tear down.
		return 1
	}
	fmt.Printf("  [OK] vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
	if len(res.StartWarnings) > 0 {
		fmt.Printf("  start warnings (recognized=%v): %v\n", res.WarningsRecognized, res.StartWarnings)
	}

	if keep {
		fmt.Printf("=== selftest=bring-up OK — guest %d KEPT (-keep) ===\n", vmid)
		return 0
	}
	// A selftest must not leave a guest running. Tear it down (out-of-band, like the spike).
	fmt.Printf("  --- teardown: destroying selftest guest %d ---\n", vmid)
	upid, err := px.DestroyLXC(ctx, vmid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy: %v (destroy vmid %d manually)\n", err, vmid)
		return 1
	}
	if upid != "" {
		if _, err := px.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
			fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy task: %v (check vmid %d)\n", err, vmid)
			return 1
		}
	}
	// GL-5: the DR bind swap created <stateDir>/guests/<vmid>/bootstrap for this SCRATCH vmid —
	// remove the agent-owned per-guest dir with the guest (never a real drive's bind source; a
	// scratch vmid has no other state). Best-effort.
	if bmode == reconcile.ModeDRGuestLoss {
		if err := os.RemoveAll(filepath.Join("/var/lib/felhom-agent/guests", strconv.Itoa(vmid))); err != nil {
			fmt.Fprintf(os.Stderr, "  [WARN] scratch mp9 host dir cleanup: %v\n", err)
		}
	}
	fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
	return 0
}

// provisionArgs bundles the --selftest=provision inputs.
type provisionArgs struct {
	archive     string
	vmid        int
	hostname    string
	customerID  string        // baked into bootstrap (the hub config-pull target)
	hubPassword string        // the customer's hub retrieval passphrase (SECRET) — baked into bootstrap
	sizing      bringUpSizing // OS-rootfs / Docker-data sizing for the bring-up half
}

// sanitizeHostname makes s a DNS-safe LXC hostname (RFC 1123 label-ish): lowercase, any run of
// invalid characters collapses to a single '-', leading/trailing '-' stripped, capped at 63 chars.
// Returns "" if nothing usable remains (caller then sets no hostname). PVE itself validates, but a
// customer id can legitimately contain characters (e.g. '_' or spaces) that a hostname cannot.
func sanitizeHostname(s string) string {
	s = strings.ToLower(strings.TrimSpace(s))
	var b strings.Builder
	prevDash := false
	for _, r := range s {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
			prevDash = false
		} else if !prevDash && b.Len() > 0 {
			b.WriteByte('-')
			prevDash = true
		}
	}
	out := strings.Trim(b.String(), "-")
	if len(out) > 63 {
		out = strings.Trim(out[:63], "-")
	}
	return out
}

// runSelftestProvision runs the FULL slice-8A provisioning chain on-demand: the slice-7 bring-up
// FRONT half (provision mode, golden) + the slice-8A BACK half (mint per-guest token → render the
// stable bootstrap.json → write 0600 → chown to the mapped guest-root → attach the read-only bind
// mount). The guest is KEPT (the golden's baked controller-bootstrap unit then deploys the baked
// controller from the mount). The per-guest token is NEVER printed (only its hash is persisted +
// the 0600 file holds the plaintext).
func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.Logger, a provisionArgs) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	if a.archive == "" || a.vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=provision requires -archive <golden volid> and -vmid <N>")
		return 2
	}
	if cfg.Backup.RestoreStorage == "" {
		fmt.Fprintln(os.Stderr, "selftest=provision requires backup.restore_storage in config")
		return 2
	}
	if a.customerID == "" || a.hubPassword == "" {
		fmt.Fprintln(os.Stderr, "selftest=provision requires -customer-id and -hub-password (the customer's hub retrieval passphrase, so the controller pulls its config)")
		return 2
	}
	if err := cfg.LocalAPI.Validate(); err != nil || !cfg.LocalAPI.Enabled() {
		fmt.Fprintln(os.Stderr, "selftest=provision requires local_api.enable + local_api.listen_addr in config:", err)
		return 2
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	// The leaf fingerprint baked into the bootstrap must be the SAME leaf the daemon's local-API
	// server serves — so use the configured (persisted) cert path. EnsureLeaf generates it once.
	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
	_, fingerprint, _, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest=provision: local-api leaf:", err)
		return 1
	}
	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest=provision: token store:", err)
		return 1
	}
	defer tokens.Close()

	// --- FRONT HALF: bring up the guest (provision mode) ---
	queue := reconcile.NewQueue()
	defer queue.Close()
	var journal *reconcile.Journal
	if jp := reconcileJournalPath(cfg); jp != "" {
		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
			if j, err := reconcile.OpenJournal(jp); err == nil {
				journal = j
				defer journal.Close()
			}
		}
	}
	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
	engine := reconcile.NewEngine(reconcile.EngineOptions{
		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
	})

	// Default the guest hostname to the customer id (DNS-safe-sanitized) when not explicitly given,
	// so the CT/LXC is named meaningfully (e.g. "demo-felhom") instead of inheriting the golden's
	// baked "felhom-golden". An explicit -hostname always wins.
	hostname := a.hostname
	if hostname == "" {
		hostname = sanitizeHostname(a.customerID)
	}

	fmt.Printf("=== felhom-agent %s selftest=provision (vmid=%d customer=%s hostname=%s) ===\n", version, a.vmid, a.customerID, hostname)
	engine.Recover(ctx)
	fmt.Printf("  --- front half: bring-up (provision) %s → vmid %d ---\n", a.archive, a.vmid)
	res := engine.RunBringUp(ctx, reconcile.BringUpSpec{
		Mode: reconcile.ModeProvision, Archive: a.archive, VMID: a.vmid,
		RestoreStorage: cfg.Backup.RestoreStorage, Hostname: hostname,
		Pool:  reconcile.DefaultPool, // provision into the felhom pool; required under the scoped token
		Cores: a.sizing.Cores, MemoryMB: a.sizing.MemoryMB,
		RootfsGrowGB: a.sizing.RootfsGrowGB, DataVolGrowGB: a.sizing.DataVolGrowGB, DataVolMount: a.sizing.DataVolMount,
		SysDataGrowGB: a.sizing.SysDataGrowGB, SysDataMount: a.sizing.SysDataMount,
		IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
	})
	if res.Err != nil || !res.Pass {
		fmt.Fprintf(os.Stderr, "  [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)
		return 1
	}
	fmt.Printf("  [OK] front half: vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)

	// --- BACK HALF: mint token + populate the bootstrap config mount (host-side, no pct exec) ---
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	bh := provision.NewBackHalf(tokens, runner, "", logger)
	fmt.Println("  --- back half: mint per-guest token + populate bootstrap config mount ---")
	pres, err := bh.Provision(ctx, provision.Input{
		VMID:        a.vmid,
		Customer:    provision.DocCustomer{ID: a.customerID},
		Hub:         provision.DocHub{URL: cfg.Hub.URL, RetrievalPassword: a.hubPassword},
		Endpoint:    cfg.LocalAPI.ListenAddr,
		Fingerprint: fingerprint,
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] back-half provision (vmid %d): %v\n", a.vmid, err)
		return 1
	}
	fmt.Printf("  [OK] back half: bootstrap mount %s → %s on vmid %d (host dir %s)\n",
		pres.MountKey, pres.GuestPath, pres.VMID, pres.HostDir)
	fmt.Printf("  local-api endpoint %s · leaf fp %s · token: minted (not printed)\n", cfg.LocalAPI.ListenAddr, fingerprint)
	fmt.Printf("=== selftest=provision OK — guest %d provisioned + bootstrap-mounted (KEPT) ===\n", a.vmid)
	fmt.Println("    next: reboot the guest → the golden's baked controller-bootstrap unit deploys the controller,")
	fmt.Println("    which PULLS its controller.yaml from the hub (retrieval passphrase) and merges in this local_api.")
	return 0
}

// escrowCeremonyOpts carries the CLI switches into the shared ceremony core (v0.88.0 extraction —
// the work is identical for both output modes; only the surfacing differs).
type escrowCeremonyOpts struct {
	storage            string
	paperkey           bool
	offline            bool
	upload             bool
	identityBundlePath string
	directivePath      string
}

// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the
// --output=json wire object with RecoveryCode left EMPTY (the shells place R themselves, once).
type escrowCeremonyOutcome struct {
	R           string
	Sum         escrow.CeremonyOutput
	OfflineCopy []byte // text-mode print block (opt-in b)
	Paperkey    string // text-mode print block (opt-in a) — SECRET-adjacent
	Posture     escrow.Posture
	Storage     string // banner data
	Identity    bool   // banner data
}

// escrowCeremonyErr classifies a ceremony failure so both output modes keep the pre-v0.88.0 exit
// codes and stderr shapes: usage → exit 2; setup/create/upload → exit 1. kind "upload" means R
// was already minted — the text shell still surfaces it before the failure (the customer must
// receive R; the blob simply never reached the hub), exactly the pre-extraction print order.
type escrowCeremonyErr struct {
	kind string // "usage" | "setup" | "create" | "upload"
	err  error
}

func (e *escrowCeremonyErr) Error() string { return e.err.Error() }

// escrowCeremony is the shared ceremony core (slice 7 + 10D.1 + fork-4, extracted v0.88.0 for the
// --output=json machine mode): resolve the key, assemble the identity bundle (WG key + staged
// restic password auto-attach), Create (R + self-verified blob), wipe the staged secret, upload
// when asked. It PRINTS NOTHING — the output-mode shells own every byte of stdout/stderr. R is
// returned for the caller to surface exactly once; escrow.Create never logs it and neither do we.
func escrowCeremony(ctx context.Context, cfg config.Config, logger *slog.Logger, opts escrowCeremonyOpts) (escrowCeremonyOutcome, *escrowCeremonyErr) {
	var out escrowCeremonyOutcome
	storage := opts.storage
	if storage == "" {
		storage = cfg.Escrow.PBSStorageID
	}
	if storage == "" {
		return out, &escrowCeremonyErr{kind: "usage", err: fmt.Errorf("selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")}
	}
	out.Storage = storage
	keyPath := cfg.Backup.PBSEncKeyPath(storage)
	if _, err := os.Stat(keyPath); err != nil {
		return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("PBS key for %q not found (%s): %v", storage, keyPath, err)}
	}

	// Slice 10D.1: optionally ALSO wrap the identity bundle under the same R, and carry the non-secret
	// directive for the hub. The bundle file is a 0600 secret (tunnel/pbs tokens); the directive is
	// non-secret (pbs repo/ns, expected fingerprint, tunnel id).
	var identity *escrow.IdentityBundle
	var directive json.RawMessage
	if opts.identityBundlePath != "" {
		raw, err := os.ReadFile(opts.identityBundlePath)
		if err != nil {
			return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("reading identity bundle %s: %v", opts.identityBundlePath, err)}
		}
		var b escrow.IdentityBundle
		if err := json.Unmarshal(raw, &b); err != nil {
			return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("identity bundle is not valid JSON {tunnel_token,pbs_token}: %v", err)}
		}
		identity = &b
		if opts.directivePath != "" {
			if d, err := os.ReadFile(opts.directivePath); err == nil && json.Valid(d) {
				directive = d
			}
		}
	}
	// S3: auto-inject the offsite WG private key into the escrowed identity when the key file
	// exists — a NEW escrow run should always capture the live tunnel identity. Field NAME only
	// in logs, never the value. No key file + no bundle flag → pre-S3 behavior, byte-compatible.
	{
		wgKeyPath := wgtunnel.KeyFilePath(cfg.WGTunnel.WithDefaults().StateDir)
		probe := identity
		if probe == nil {
			probe = &escrow.IdentityBundle{}
		}
		attached, err := escrow.AttachWGKey(probe, wgKeyPath)
		if err != nil {
			return out, &escrowCeremonyErr{kind: "setup", err: err}
		}
		if attached {
			identity = probe
			logger.Info("escrow: identity bundle: +wg_private_key")
		}
	}
	// fork-4: auto-inject the staged offsite restic repo password (controller-pushed via the local API)
	// into the escrowed identity, so DR can recover the offsite DATA key with the one recovery code R.
	// Field NAME only in logs. No staged file → clean no-attach (pre-fork-4 behavior). Wiped after create.
	resticStaged := false
	resticPwSHA256 := "" // SLICE 3: sha256 of the sealed password (safe to upload/serve; value never logged)
	{
		probe := identity
		if probe == nil {
			probe = &escrow.IdentityBundle{}
		}
		attached, err := escrow.AttachResticPassword(probe, escrow.StagedResticPasswordPath())
		if err != nil {
			return out, &escrowCeremonyErr{kind: "setup", err: err}
		}
		if attached {
			identity = probe
			resticStaged = true
			// Hash EXACTLY the value sealed into the blob — the controller matches this against
			// sha256(its local repo_password) to auto-confirm the escrow (hub-verified, SLICE 3).
			resticPwSHA256 = escrow.HashResticPassword(probe.ResticRepoPassword)
			logger.Info("escrow: identity bundle: +restic_repo_password")
		}
	}
	out.Identity = identity != nil

	// NB: nothing about R is logged. The logger never sees R; only the shells surface it, once.
	logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath, "with_identity", identity != nil)

	R, res, err := escrow.Create(ctx, escrow.CreateOptions{
		KeyPath:         keyPath,
		Posture:         escrow.Posture(cfg.Escrow.Posture),
		WantOfflineCopy: opts.offline,
		WantPaperkey:    opts.paperkey,
		IdentityBundle:  identity,
	})
	if err != nil {
		return out, &escrowCeremonyErr{kind: "create", err: err}
	}
	// fork-4: the staged restic password is now sealed inside the R-wrapped blob — wipe the transient
	// 0600 staging file so it never lingers on disk (field name only; a wipe failure is a loud warn).
	if resticStaged {
		if werr := escrow.WipeStagedResticPassword(); werr != nil {
			logger.Warn("escrow: could not wipe the staged restic password after create", "err", werr)
		}
	}

	out.R = R
	out.Posture = res.Posture
	out.OfflineCopy = res.OfflineCopy
	out.Paperkey = res.Paperkey
	out.Sum = escrow.CeremonyOutput{
		Version:           escrow.CeremonyOutputVersion,
		KeyFingerprint:    res.KeyFingerprint,
		EntropyBits:       res.EntropyBits,
		BlobBytes:         len(res.Blob),
		IdentityBlobBytes: len(res.IdentityBlob),
		ResticPwSealed:    resticStaged,
	}
	if opts.upload {
		if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil {
			// R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets
			// the text shell keep the pre-extraction order (R surfaced, THEN the failure).
			return out, &escrowCeremonyErr{kind: "upload", err: err}
		}
		out.Sum.Uploaded = true
	}
	return out, nil
}

// printEscrowTextBanner prints the text-mode header line (shared by the success and the
// post-banner failure paths so the pre-extraction output stays byte-identical — including the
// posture: the historical banner always printed the DEFAULT posture, not the result's).
func printEscrowTextBanner(out escrowCeremonyOutcome) {
	fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, out.Storage, escrow.DefaultPosture, out.Identity)
}

// printEscrowTextRBlock surfaces R EXACTLY ONCE — to stdout, with the write-it-down banner —
// followed by the non-secret blob facts and the opt-in print blocks. Never logged/persisted.
func printEscrowTextRBlock(out *escrowCeremonyOutcome) {
	fmt.Println()
	fmt.Println("  ┌──────────────────────────────────────────────────────────────────────┐")
	fmt.Println("  │  RECOVERY CODE — write it down now. It is shown ONCE and never stored.  │")
	fmt.Println("  │  Without it your offsite backups are unrecoverable, by anyone.          │")
	fmt.Println("  └──────────────────────────────────────────────────────────────────────┘")
	fmt.Println("     " + out.R)
	fmt.Println()
	out.R = "" // drop our reference promptly

	fmt.Printf("  blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n",
		out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Posture, out.Sum.EntropyBits)
	fmt.Println("  self-verify: the blob unwraps back to the key with R (recoverability confirmed)")

	if len(out.OfflineCopy) > 0 {
		fmt.Println("  --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---")
		fmt.Println(base64.StdEncoding.EncodeToString(out.OfflineCopy))
	}
	if out.Paperkey != "" {
		fmt.Println("  --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---")
		fmt.Println(out.Paperkey)
	}

	if out.Sum.IdentityBlobBytes > 0 {
		fmt.Printf("  identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", out.Sum.IdentityBlobBytes)
	}
}

// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
// blob. With -upload it PUTs the opaque blob to the hub. Enrollment-time, root-capable (reads the
// 0600 key). Output modes (v0.88.0):
//   - text (default): the historical human output, byte-identical to pre-v0.88.0 — R to stdout
//     EXACTLY ONCE inside the write-it-down banner (never to the logger/journald).
//   - json: ONE machine-readable JSON object on stdout (escrow.CeremonyOutput — carries R) and
//     NOTHING else there; every human/info line goes to stderr; failures exit non-zero with no
//     partial JSON. This is the controller-driven ceremony's parse surface (spike §2.3: the text
//     banner is positionally brittle).
func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath, outputMode string) int {
	switch outputMode {
	case "", "text", "json":
	default:
		fmt.Fprintf(os.Stderr, "selftest=escrow-create: unknown -output %q (text|json)\n", outputMode)
		return 2
	}
	jsonMode := outputMode == "json"
	if jsonMode && (offline || paperkey) {
		// The opt-in print blocks are PRINT-oriented (base64 blob / paperkey text on stdout) —
		// in json mode stdout carries exactly one JSON object, so refuse loudly instead of
		// silently dropping what the caller asked for.
		fmt.Fprintln(os.Stderr, "selftest=escrow-create: -offline/-paperkey are text-mode only (their output is print-oriented)")
		return 2
	}

	out, cerr := escrowCeremony(ctx, cfg, logger, escrowCeremonyOpts{
		storage: storage, paperkey: paperkey, offline: offline, upload: upload,
		identityBundlePath: identityBundlePath, directivePath: directivePath,
	})
	if cerr != nil {
		switch cerr.kind {
		case "usage":
			fmt.Fprintln(os.Stderr, cerr.err)
			return 2
		case "setup":
			fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", cerr.err)
			return 1
		case "create":
			if !jsonMode {
				printEscrowTextBanner(out)
			}
			fmt.Fprintln(os.Stderr, "  [FAIL] escrow create:", cerr.err)
			return 1
		default: // "upload" — R was minted; text mode still surfaces it (pre-extraction order)
			if !jsonMode {
				printEscrowTextBanner(out)
				printEscrowTextRBlock(&out)
			}
			out.R = ""
			fmt.Fprintln(os.Stderr, "  [FAIL] upload escrow to hub:", cerr.err)
			return 1
		}
	}

	if jsonMode {
		// Human/info lines → stderr (the job runner's diagnostics tail); the ONE JSON object with
		// R → stdout. The consumer (localapi ceremony job) extracts R and zeroes its buffers.
		fmt.Fprintf(os.Stderr, "=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v output=json) ===\n", version, out.Storage, out.Posture, out.Identity)
		fmt.Fprintf(os.Stderr, "  blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · ~%.0f bits R · self-verify OK\n",
			out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Sum.EntropyBits)
		if out.Sum.IdentityBlobBytes > 0 {
			fmt.Fprintf(os.Stderr, "  identity escrow: %d bytes · restic_pw_sealed=%v\n", out.Sum.IdentityBlobBytes, out.Sum.ResticPwSealed)
		}
		if out.Sum.Uploaded {
			fmt.Fprintln(os.Stderr, "  uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
		}
		wire := out.Sum
		wire.RecoveryCode = out.R
		if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil {
			fmt.Fprintf(os.Stderr, "selftest=escrow-create: emitting JSON: %v\n", err)
			return 1
		}
		// Best-effort scrub: drop every R reference promptly. Go's GC may retain stale copies of
		// the string backing array — this shrinks the window, it cannot guarantee erasure.
		wire.RecoveryCode = ""
		out.R = ""
		return 0
	}

	printEscrowTextBanner(out)
	printEscrowTextRBlock(&out)
	if out.Sum.Uploaded {
		fmt.Println("  uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
	}
	fmt.Println("=== selftest=escrow-create OK ===")
	return 0
}

// runSelftestEscrowConsume exercises the slice-10C production Consume path live: recover K from an
// R-wrapped blob, gate it on the expected fingerprint, install it at -keydest. R is taken BY HAND
// from the env var FELHOM_RECOVERY_CODE (kept off the command line / ps) — never a flag, never
// logged. The other three inputs (blob/fingerprint/keydest) are flags (10D sources blob+fingerprint
// from the hub directive). This is the real Consume code, not a throwaway harness.
func runSelftestEscrowConsume(ctx context.Context, logger *slog.Logger, blobPath, expectedFP, keyDest string) int {
	if blobPath == "" || expectedFP == "" || keyDest == "" {
		fmt.Fprintln(os.Stderr, "selftest=escrow-consume requires -blob, -fingerprint and -keydest (R via env FELHOM_RECOVERY_CODE)")
		return 2
	}
	R := os.Getenv("FELHOM_RECOVERY_CODE")
	if R == "" {
		fmt.Fprintln(os.Stderr, "selftest=escrow-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input; never a flag/arg)")
		return 2
	}
	blob, err := os.ReadFile(blobPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "selftest=escrow-consume: reading blob %s: %v\n", blobPath, err)
		return 1
	}
	fmt.Printf("=== felhom-agent %s selftest=escrow-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
	logger.Info("escrow: consuming R-wrapped escrow (Unwrap → fingerprint-gate → install)",
		"blob_bytes", len(blob), "key_dest", keyDest) // R is NOT logged
	if err := escrow.Consume(ctx, blob, R, expectedFP, keyDest); err != nil {
		R = ""                                            // drop the reference
		fmt.Fprintln(os.Stderr, "  [FAIL] consume:", err) // the error never contains R or key bytes
		return 1
	}
	R = "" // drop the reference promptly
	fmt.Printf("  [OK] recovered key installed at %s (fingerprint-gated, 0600) — ready for the PBS restore\n", keyDest)
	fmt.Println("=== selftest=escrow-consume OK ===")
	return 0
}

// runSelftestIdentityConsume recovers the IDENTITY bundle from its age blob with R (slice 10D.1/10D.3)
// and writes the recovered {tunnel_token, pbs_token} JSON to -keydest (0600). R is taken BY HAND from
// FELHOM_RECOVERY_CODE (off the command line); the recovered tokens are never logged. The drill then
// uses the tunnel token to re-establish the tunnel + the pbs token for steady-state.
func runSelftestIdentityConsume(ctx context.Context, cfg config.Config, logger *slog.Logger, blobPath, keyDest string, installWGKey bool) int {
	if blobPath == "" || keyDest == "" {
		fmt.Fprintln(os.Stderr, "selftest=identity-consume requires -blob and -keydest (R via env FELHOM_RECOVERY_CODE)")
		return 2
	}
	R := os.Getenv("FELHOM_RECOVERY_CODE")
	if R == "" {
		fmt.Fprintln(os.Stderr, "selftest=identity-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input)")
		return 2
	}
	blob, err := os.ReadFile(blobPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "selftest=identity-consume: reading blob %s: %v\n", blobPath, err)
		return 1
	}
	fmt.Printf("=== felhom-agent %s selftest=identity-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
	logger.Info("escrow: recovering identity bundle from R-wrapped age blob", "blob_bytes", len(blob)) // R + tokens NOT logged
	bundle, err := escrow.UnwrapIdentityBundle(ctx, blob, R)
	if err != nil {
		R = ""
		fmt.Fprintln(os.Stderr, "  [FAIL] identity consume:", err) // never contains R or token bytes
		return 1
	}
	R = ""
	raw, _ := json.Marshal(bundle)
	if err := os.WriteFile(keyDest, raw, 0o600); err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] writing recovered bundle:", err)
		return 1
	}
	// R-199 / §8.6: this line used to read "(tunnel_token + pbs_token)" — an enumeration that was
	// accurate when it was written (pre-fork-4) and became a MISSTATEMENT the moment v0.77.0 sealed the
	// offsite repository password into the same bundle. Anyone reading the old output would conclude the
	// repository password was not there, and that is part of how the chain's extraction link came to be
	// described as missing for a month. Name what was recovered from THIS bundle, and name what is
	// absent, rather than reciting a fixed list.
	recovered := []string{"tunnel_token", "pbs_token"}
	var absent []string
	if bundle.WGPrivateKey != "" {
		recovered = append(recovered, "wg_private_key")
	} else {
		absent = append(absent, "wg_private_key")
	}
	if bundle.ResticRepoPassword != "" {
		recovered = append(recovered, "restic_repo_password")
	} else {
		absent = append(absent, "restic_repo_password (pre-fork-4 blob — the field did not exist when this was sealed)")
	}
	fmt.Printf("  [OK] identity recovered (%s) → %s (0600) — values never printed\n", strings.Join(recovered, " + "), keyDest)
	if len(absent) > 0 {
		fmt.Printf("  [NOTE] fields ABSENT from this bundle: %s\n", strings.Join(absent, "; "))
	}

	// S5 DR: install the recovered WG private key so the tunnel re-establishes with the SAME
	// identity/pubkey (→ the same hub /32), no fresh keygen. Create-only (refuses to overwrite a
	// present key). The VALUE is never printed — field NAME only.
	if installWGKey {
		stateDir := cfg.WGTunnel.WithDefaults().StateDir
		switch {
		case bundle.WGPrivateKey == "":
			fmt.Printf("  [WARN] -install-wg-key set but the recovered bundle has NO wg_private_key (pre-S3 blob) — "+
				"DR falls back to fresh keygen + re-register (keeps the /32 via hub re-key-in-place). key path: %s\n",
				wgtunnel.KeyFilePath(stateDir))
		default:
			if err := wgtunnel.InstallRecoveredKey(stateDir, bundle.WGPrivateKey); err != nil {
				fmt.Fprintln(os.Stderr, "  [FAIL] install recovered wg key:", err) // never contains the key value
				return 1
			}
			logger.Info("escrow: installed recovered identity field", "field", "wg_private_key",
				"key_path", wgtunnel.KeyFilePath(stateDir)) // NAME only, never the value
			fmt.Printf("  [OK] recovered wg_private_key installed at %s (0600, create-only) — tunnel will re-establish with the same identity\n",
				wgtunnel.KeyFilePath(stateDir))
		}
	}
	fmt.Println("=== selftest=identity-consume OK ===")
	return 0
}

// escrowUploadRequest is the agent→hub wire shape for the opaque escrow blob. MUST stay in lockstep
// with the hub's ingest struct (felhom-hub api.escrowUploadRequest). The hub stores the bytes and
// never decrypts them.
type escrowUploadRequest struct {
	BlobB64        string `json:"blob_b64"`        // base64 of the opaque R-wrapped blob (ciphertext)
	KeyFingerprint string `json:"key_fingerprint"` // for operator display only
	Posture        string `json:"posture"`         // e.g. "zero_knowledge"
	// Slice 10D.1 — optional DR bundle (identity escrow + non-secret directive). Omitted in slice-7.
	IdentityBlobB64 string          `json:"identity_blob_b64,omitempty"`
	DirectiveJSON   json.RawMessage `json:"directive,omitempty"`
	CreatedAt       string          `json:"created_at"` // RFC3339
	// SLICE 3 — sha256 hex of the offsite restic repo password sealed in the identity blob (present only
	// when a staged password was folded in). Non-reversible hash of a 256-bit random secret — safe to
	// store/serve; lets the controller VERIFY "the escrow covers the CURRENT key" and auto-confirm.
	ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"`
}

// uploadEscrowBlob PUTs the opaque blob (and, for 10D, the identity blob + non-secret directive) to
// the hub, authed with the per-host key. The hub stores ciphertext + non-secret fields; no usable
// secret leaves the agent.
func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateResult, directive json.RawMessage, resticPwSHA256 string) error {
	if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
		return fmt.Errorf("hub not configured (url/host_id/api_key)")
	}
	upReq := escrowUploadRequest{
		BlobB64:        base64.StdEncoding.EncodeToString(res.Blob),
		KeyFingerprint: res.KeyFingerprint,
		Posture:        string(res.Posture),
		CreatedAt:      time.Now().UTC().Format(time.RFC3339),
		ResticPwSHA256: resticPwSHA256, // "" when no staged password was folded in → omitted on the wire
	}
	if len(res.IdentityBlob) > 0 {
		upReq.IdentityBlobB64 = base64.StdEncoding.EncodeToString(res.IdentityBlob)
		upReq.DirectiveJSON = directive
	}
	body, _ := json.Marshal(upReq)
	url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
	req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+cfg.Hub.APIKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
	}
	return nil
}

// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
// path), then lists + prints the resulting PBSSnapshot records (verify-state included).
// Standalone on the host. Covers the runbook's (c) verify and (d) list.
func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
		return 1
	}
	px, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version)
	targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "  [FAIL] discover pbs storages:", err)
		return 1
	}
	if len(targets) == 0 {
		fmt.Println("  no pbs storages configured on this host")
		return 0
	}
	store := pbs.NewSnapshotStore()
	loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
		Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil },
		Store:   store,
		Logger:  logger,
	})
	// One synchronous verify+list pass over all datastores.
	loop.RunOnce(ctx)
	snaps := store.PBSSnapshots(ctx)
	printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps)
	failed := 0
	for _, s := range snaps {
		if s.VerifyState == pbs.VerifyFailed {
			failed++
		}
	}
	if failed > 0 {
		fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed)
		return 1
	}
	fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets))
	return 0
}

// printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
func printJSON(label string, v any) {
	if b, err := json.MarshalIndent(v, "  ", "  "); err == nil {
		fmt.Printf("  --- %s ---\n  %s\n", label, string(b))
	}
}

// runSelftestRead loads config, builds the API client, and runs the read-only
// queries against the live host, printing a short health report. It mutates
// nothing. Missing/invalid config is reported cleanly (no panic).
func runSelftestRead(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
		return 1
	}
	logger.Info("selftest (read-only) starting", "config", fmt.Sprintf("%+v", cfg.Redacted().Proxmox))

	client, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: client init:", err)
		return 1
	}

	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	fmt.Printf("=== felhom-agent %s selftest (read-only) ===\n", version)
	fmt.Printf("endpoint : %s  node=%s\n", cfg.Proxmox.Endpoint, cfg.Proxmox.Node)

	fail := 0
	report := func(label string, err error) bool {
		if err != nil {
			fmt.Printf("  [FAIL] %-14s %v\n", label, err)
			fail++
			return false
		}
		return true
	}

	if v, err := client.Version(ctx); report("version", err) {
		fmt.Printf("  [ ok ] %-14s PVE %s (release %s)\n", "version", v.Version, v.Release)
	}
	if nodes, err := client.Nodes(ctx); report("nodes", err) {
		fmt.Printf("  [ ok ] %-14s %d node(s)\n", "nodes", len(nodes))
		for _, n := range nodes {
			marker := "  "
			if n.Node == cfg.Proxmox.Node {
				marker = "* "
			}
			fmt.Printf("           %s%s status=%s fp=%s…\n", marker, n.Node, n.Status, head(n.SSLFingerprint, 17))
		}
	}
	if s, err := client.NodeStatus(ctx); report("node status", err) {
		fmt.Printf("  [ ok ] %-14s up %s, load %v, mem %s/%s, root %s/%s\n", "node status",
			dur(s.Uptime), s.LoadAvg,
			gib(s.Memory.Used), gib(s.Memory.Total), gib(s.RootFS.Used), gib(s.RootFS.Total))
	}
	if gs, err := client.ListLXC(ctx); report("list lxc", err) {
		fmt.Printf("  [ ok ] %-14s %d guest(s)\n", "list lxc", len(gs))
		for _, g := range gs {
			fmt.Printf("           - %d %q status=%s\n", g.VMID, g.Name, g.Status)
		}
	}
	// A1 (v0.62.0): the stale-lock reaper's ownership registry — needs Pool.Audit (host-install v1.9.0+).
	if p, err := client.Pool(ctx, reconcile.DefaultPool); report("pool read", err) {
		fmt.Printf("  [ ok ] %-14s pool %q, %d member(s)\n", "pool read", p.PoolID, len(p.Members))
		for _, m := range p.Members {
			if m.VMID != 0 && m.Type != "storage" {
				fmt.Printf("           - %d type=%s\n", m.VMID, m.Type)
			}
		}
	}
	if ss, err := client.NodeStorage(ctx); report("storage", err) {
		fmt.Printf("  [ ok ] %-14s %d store(s)\n", "storage", len(ss))
		for _, s := range ss {
			fmt.Printf("           - %-10s type=%-8s content=%s used=%s/%s\n",
				s.Storage, s.Type, s.Content, gib(s.Used), gib(s.Total))
		}
	}

	if fail > 0 {
		fmt.Printf("=== selftest FAILED (%d check(s)) ===\n", fail)
		return 1
	}
	fmt.Println("=== selftest OK ===")
	return 0
}

// runSelftestTask exercises WaitTask on a reversible op against -vmid: snapshot ->
// rollback -> delete-snapshot. Explicitly gated; never runs under bare --selftest.
func runSelftestTask(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if err := cfg.Validate(); err != nil {
		fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
		return 1
	}
	if vmid == 0 {
		fmt.Fprintln(os.Stderr, "selftest=task requires -vmid N (a guest safe to snapshot/rollback)")
		return 2
	}
	client, err := newProxmoxClient(cfg)
	if err != nil {
		fmt.Fprintln(os.Stderr, "selftest: client init:", err)
		return 1
	}

	// Ctrl-C aborts the wait cleanly.
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()

	const snap = "felhom-selftest"
	steps := []struct {
		name string
		do   func() (string, error)
	}{
		{"snapshot", func() (string, error) { return client.Snapshot(ctx, vmid, snap, "felhom-agent selftest") }},
		{"rollback", func() (string, error) { return client.Rollback(ctx, vmid, snap) }},
		{"delete-snapshot", func() (string, error) { return client.DeleteSnapshot(ctx, vmid, snap) }},
	}
	fmt.Printf("=== felhom-agent selftest=task (vmid %d, snapshot %q) ===\n", vmid, snap)
	for _, st := range steps {
		upid, err := st.do()
		if err != nil {
			fmt.Printf("  [FAIL] %-16s %v\n", st.name, err)
			return 1
		}
		fmt.Printf("  ....   %-16s upid=%s\n", st.name, upid)
		status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
		if err != nil {
			fmt.Printf("  [FAIL] %-16s %v\n", st.name, err)
			return 1
		}
		fmt.Printf("  [ ok ] %-16s exitstatus=%s\n", st.name, status.ExitStatus)
	}

	// Reversible SetConfig exercise: this is the first live use of the VM.Config.*
	// privilege cluster. It round-trips the cosmetic `description` field (no runtime
	// effect, fully reversible) to prove SetConfig works under the scoped token
	// before slice 4's reconcile is built on top of it.
	if rc := selftestSetConfig(ctx, client, vmid); rc != 0 {
		return rc
	}

	fmt.Println("=== selftest=task OK ===")
	return 0
}

// selftestSetConfig performs a reversible write+revert of the LXC `description`
// field on vmid and asserts both land. Returns 0 on success, non-zero (with a
// printed [FAIL] line) on any failure — the caller stops on a non-zero return.
func selftestSetConfig(ctx context.Context, client *proxmox.Client, vmid int) int {
	// 1. Read current state; capture the original description (may be absent).
	cfg, err := client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig: %v\n", "setconfig", err)
		return 1
	}
	origDesc, origPresent, err := extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description: %v\n", "setconfig", err)
		return 1
	}
	// PVE normalizes `description` by appending a trailing newline on read, so all
	// comparisons here use the shared reconcile.NormDescription (strip trailing
	// newlines) and restores write the normalized original — otherwise an exact-match
	// check sees false drift. Same helper the slice-4 reconciler uses to normalize
	// description, so the quirk has one source of truth.
	origDesc = reconcile.NormDescription(origDesc)

	// 2. Write the marker.
	marker := "felhom-selftest " + time.Now().UTC().Format(time.RFC3339)
	if rc := applySetConfig(ctx, client, vmid, "setconfig", map[string]string{"description": marker}); rc != 0 {
		return rc
	}

	// 3. Verify the marker landed.
	cfg, err = client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig (verify): %v\n", "setconfig", err)
		return 1
	}
	got, present, err := extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description (verify): %v\n", "setconfig", err)
		return 1
	}
	if !present || reconcile.NormDescription(got) != marker {
		fmt.Printf("  [FAIL] %-16s write did not land: present=%v got=%q want=%q\n", "setconfig", present, got, marker)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s description verified == marker\n", "verify-write")

	// 4. Restore the original value (or clear it if it was absent originally).
	var revert map[string]string
	if origPresent {
		revert = map[string]string{"description": origDesc}
	} else {
		revert = map[string]string{"delete": "description"}
	}
	if rc := applySetConfig(ctx, client, vmid, "setconfig-revert", revert); rc != 0 {
		return rc
	}

	// 5. Confirm the restore.
	cfg, err = client.GuestConfig(ctx, vmid)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s GuestConfig (revert verify): %v\n", "setconfig-revert", err)
		return 1
	}
	got, present, err = extraString(cfg, "description")
	if err != nil {
		fmt.Printf("  [FAIL] %-16s decode description (revert verify): %v\n", "setconfig-revert", err)
		return 1
	}
	if origPresent {
		if !present || reconcile.NormDescription(got) != origDesc {
			fmt.Printf("  [FAIL] %-16s revert did not restore: present=%v got=%q want=%q\n", "setconfig-revert", present, reconcile.NormDescription(got), origDesc)
			return 1
		}
	} else if present {
		fmt.Printf("  [FAIL] %-16s revert did not clear: still present got=%q\n", "setconfig-revert", got)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s description restored to original\n", "verify-revert")
	return 0
}

// applySetConfig runs one SetConfig and asserts success, handling PVE's dual-mode
// return: a UPID means async (WaitTask + assert exitstatus OK); an empty string
// means PVE applied it synchronously (not an error — phase1-2/mutate.go contract).
func applySetConfig(ctx context.Context, client *proxmox.Client, vmid int, step string, params map[string]string) int {
	upid, err := client.SetConfig(ctx, vmid, params)
	if err != nil {
		fmt.Printf("  [FAIL] %-16s %v\n", step, err)
		return 1
	}
	if upid == "" {
		// Synchronous path: empty UPID is a clean success.
		fmt.Printf("  [ ok ] %-16s synchronous  exitstatus=OK\n", step)
		return 0
	}
	status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
	if err != nil {
		fmt.Printf("  [FAIL] %-16s %s  %v\n", step, upid, err)
		return 1
	}
	if status.ExitStatus != "OK" {
		fmt.Printf("  [FAIL] %-16s %s  exitstatus=%s\n", step, upid, status.ExitStatus)
		return 1
	}
	fmt.Printf("  [ ok ] %-16s %s  exitstatus=%s\n", step, upid, status.ExitStatus)
	return 0
}

// extraString reads a string-valued key from GuestConfig.Extra (raw JSON). It
// returns ("", false, nil) when the key is absent, and decodes the JSON string
// otherwise.
func extraString(cfg proxmox.GuestConfig, key string) (string, bool, error) {
	raw, ok := cfg.Extra[key]
	if !ok || len(raw) == 0 {
		return "", false, nil
	}
	var s string
	if err := json.Unmarshal(raw, &s); err != nil {
		return "", false, err
	}
	return s, true, nil
}

// --- small helpers / flag type ---

func envOr(key, def string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return def
}

func head(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n]
}

func dur(seconds int64) string { return (time.Duration(seconds) * time.Second).String() }

func gib(bytes int64) string { return fmt.Sprintf("%.1fGiB", float64(bytes)/(1<<30)) }

// runSelftestLANResolver ensures dnsmasq + the host base config, then applies the split-horizon record
// for ONE guest (its live LAN IP + domain discovered from the running guest). Prints what it wrote;
// verify the actual resolution out-of-band (dig @<host-ip> <app>.<domain>). Requires a host LAN IP
// (lan_resolver.host_ip or derivable from local_api.listen_addr).
func runSelftestLANResolver(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
	if vmid <= 0 {
		fmt.Fprintln(os.Stderr, "selftest=lanresolver: -vmid is required")
		return 1
	}
	lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
	if lr.HostIP == "" {
		fmt.Fprintln(os.Stderr, "selftest=lanresolver: no host IP (set lan_resolver.host_ip or local_api.listen_addr)")
		return 1
	}
	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
	if mode == "" {
		mode = proxmox.RunnerSudo
	}
	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
	mgr := lanresolver.NewManager(runner, lr.HostIP, lr.Upstreams, logger)

	fmt.Printf("=== felhom-agent %s selftest=lanresolver (vmid=%d host_ip=%s) ===\n", version, vmid, lr.HostIP)
	if err := mgr.EnsureDnsmasq(ctx); err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] EnsureDnsmasq: %v\n", err)
		return 1
	}
	fmt.Printf("  [OK] dnsmasq present + base config (listen %s, upstreams %v)\n", lr.HostIP, lr.Upstreams)
	cid := lanresolver.CustomerID(lr.StateDir, vmid)
	if err := mgr.ReconcileGuest(ctx, vmid, cid); err != nil {
		fmt.Fprintf(os.Stderr, "  [FAIL] ReconcileGuest: %v\n", err)
		return 1
	}
	fmt.Printf("  [OK] split-horizon applied for guest %d (customer=%q). Verify: dig @%s felhom.<domain>\n", vmid, cid, lr.HostIP)
	return 0
}

// selftestFlag is a flag.Value that also satisfies IsBoolFlag, so `--selftest`
// works bare (read-only) and `--selftest=task` / `--selftest=read` set the mode.
type selftestFlag struct{ mode string }

func (f *selftestFlag) String() string   { return f.mode }
func (f *selftestFlag) IsBoolFlag() bool { return true }
func (f *selftestFlag) Set(v string) error {
	switch v {
	case "true", "", "read":
		f.mode = "read"
	case "task":
		f.mode = "task"
	case "hub":
		f.mode = "hub"
	case "storage":
		f.mode = "storage"
	case "backup":
		f.mode = "backup"
	case "restore-test":
		f.mode = "restore-test"
	case "restore-test-due":
		f.mode = "restore-test-due"
	case "pbs-verify":
		f.mode = "pbs-verify"
	case "lanresolver":
		f.mode = "lanresolver"
	case "bring-up":
		f.mode = "bring-up"
	case "provision":
		f.mode = "provision"
	case "escrow-create":
		f.mode = "escrow-create"
	case "escrow-consume":
		f.mode = "escrow-consume"
	case "identity-consume":
		f.mode = "identity-consume"
	case "controller-swap":
		f.mode = "controller-swap"
	default:
		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap)", v)
	}
	return nil
}