admin
ed97232598
Implements SPIKE-smart-coverage-2026-07-25 fixes B+A (additive; MinAgent unchanged). Fix B: storage.SmartReader.SMARTForBacking wired into the /disks union path (localapi Smart seam) so registry/USB drives get a real SMART read (watchdog Known stays enrich-free). Fix A: smartDeviceFor resolves dm/LVM to the whole disk via /sys/block/<dm>/slaves (recursive; skips >1-disk); the builtin local dir on the LVM root gets a SMART-only device from its containing filesystem (never touches backing/durable_id). SmartSummary.ModelName captured from smartctl. Fix C (-d sat) stays rejected. Tests + red-proofs (dm multi-disk skip, enrich smartHint, union routing); Known-path-never-SMARTs asserted.
2824 lines
130 KiB
Go
2824 lines
130 KiB
Go
// Command felhom-agent is the host agent.
|
|
//
|
|
// With no --selftest flag it runs as the daemon: the host-report poll loop
|
|
// (slice 3) that periodically POSTs a read-only host-report to the hub (the
|
|
// heartbeat). --selftest=read|task exercise the proxmox layer; --selftest=hub does
|
|
// one collect+report against the hub and prints what it would send.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"net/netip"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/desired"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/dr"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/fasttick"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/guestnet"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
|
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/mgmtplane"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/pbsdr"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/poke"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/selfheal"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
|
)
|
|
|
|
// version is the agent version. Overridable at build time with
|
|
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
|
var version = "0.92.1"
|
|
|
|
// runGuestHook is the PVE hook body (`felhom-agent guest-hook <vmid> <phase>`). On pre-start it
|
|
// creates placeholder dirs for any absent bind-mount source so the guest always boots (the C1 net);
|
|
// on post-start it re-arms idle NAS automount triggers so the fresh guest namespace sees network
|
|
// shares again (RCA fix 1 — a new namespace inherits real mounts but not idle autofs triggers).
|
|
// It ALWAYS returns cleanly (exit 0) — a hook must never block a guest start. Output is written to
|
|
// stderr (PVE captures hook output into the task log).
|
|
func runGuestHook(args []string) {
|
|
if len(args) < 2 {
|
|
return
|
|
}
|
|
vmid, phase := args[0], args[1]
|
|
switch phase {
|
|
case guesthook.PhasePreStart:
|
|
// F10/rc255 (CAMPAIGN-3): a pre-start hook that panics or hangs would BLOCK the guest start (the
|
|
// campaign saw a guest bricked while a unit sat start-limited). Every phase runs recover-wrapped
|
|
// under a hard timeout and this function ALWAYS returns cleanly (main then exits 0).
|
|
runHookPhase(vmid, "pre-start", 30*time.Second, func(context.Context) {
|
|
created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
|
|
if len(created) > 0 {
|
|
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
|
|
}
|
|
})
|
|
case guesthook.PhasePostStart:
|
|
runHookPhase(vmid, "post-start", 30*time.Second, func(ctx context.Context) {
|
|
postStartNetworkReassertFn(ctx, vmid)
|
|
})
|
|
}
|
|
}
|
|
|
|
// runHookPhase runs one guest-hook phase body under a hard timeout with a panic recover, so NO phase
|
|
// can ever fail the guest start (CAMPAIGN-3 F10/rc255). A panic is logged to the PVE task log (stderr)
|
|
// and swallowed; a body that overruns the timeout is abandoned (its context is cancelled) while the
|
|
// hook returns. This is the Go belt; the wrapper script's `|| true; exit 0` is the shell belt.
|
|
func runHookPhase(vmid, phase string, timeout time.Duration, body func(context.Context)) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s PANIC recovered (boot continues): %v\n", vmid, phase, r)
|
|
}
|
|
close(done)
|
|
}()
|
|
body(ctx)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
case <-ctx.Done():
|
|
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s timed out after %s (boot continues)\n", vmid, phase, timeout)
|
|
}
|
|
}
|
|
|
|
// postStartNetworkReassertFn is the post-start hook action (seam — tests assert the wiring without
|
|
// touching real systemctl).
|
|
var postStartNetworkReassertFn = guesthook.PostStartNetworkReassert
|
|
|
|
func main() {
|
|
// Pre-start self-heal hook entrypoint. PVE invokes the registered hookscript as
|
|
// `<bin> guest-hook <vmid> <phase>`. Handled BEFORE flag parsing — it takes positional args, must be
|
|
// fast, needs no config/daemon/network, and must NEVER exit nonzero (a hook that fails would block
|
|
// the guest start). See internal/guesthook (the C1 net).
|
|
if len(os.Args) >= 2 && os.Args[1] == "guest-hook" {
|
|
runGuestHook(os.Args[2:])
|
|
return
|
|
}
|
|
var (
|
|
cfgPath string
|
|
selftest selftestFlag
|
|
vmid int
|
|
watch time.Duration
|
|
archive string
|
|
mode string
|
|
hostname string
|
|
keep bool
|
|
rootfsGrow int
|
|
dataVolGrow int
|
|
dataVolMount string
|
|
sysDataGrow int
|
|
sysDataMount string
|
|
cores int
|
|
memoryMB int
|
|
pbsStorage string
|
|
paperkey bool
|
|
offline bool
|
|
upload bool
|
|
custID string
|
|
custDomain string
|
|
custName string
|
|
custEmail string
|
|
hubPassword string
|
|
blobPath string
|
|
expectedFP string
|
|
keyDest string
|
|
installWGKey bool
|
|
idBundlePath string
|
|
directivePath string
|
|
swapImage string
|
|
outputMode string
|
|
showVersion bool
|
|
)
|
|
flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
|
|
flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-sysdata-grow/-cores/-memory; keeps the guest)")
|
|
flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
|
|
flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
|
|
flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
|
|
flag.StringVar(&mode, "mode", "provision", "for --selftest=bring-up: `provision` (golden, fresh identity) | `dr` (customer backup, preserve continuity)")
|
|
flag.StringVar(&hostname, "hostname", "", "for --selftest=bring-up provision: the hostname to set on the new guest")
|
|
flag.BoolVar(&keep, "keep", false, "for --selftest=bring-up: KEEP the guest instead of tearing it down at the end")
|
|
flag.IntVar(&rootfsGrow, "rootfs-grow", 0, "for --selftest=bring-up|provision: grow the OS rootfs by this many GiB after restore (0 = keep golden size)")
|
|
flag.IntVar(&dataVolGrow, "datavol-grow", 0, "for --selftest=bring-up|provision: grow the golden's Docker-data volume (mp0) by this many GiB (0 = keep golden size)")
|
|
flag.StringVar(&dataVolMount, "datavol-mount", "", "for --selftest=bring-up|provision: the mpN slot of the Docker-data volume to grow (default mp0)")
|
|
flag.IntVar(&sysDataGrow, "sysdata-grow", 0, "for --selftest=bring-up|provision: grow the golden's SSD user-data volume (mp1, /mnt/sys_drive) by this many GiB (0 = keep golden size)")
|
|
flag.StringVar(&sysDataMount, "sysdata-mount", "", "for --selftest=bring-up|provision: the mpN slot of the user-data volume to grow (default mp1)")
|
|
flag.IntVar(&cores, "cores", 0, "for --selftest=bring-up|provision: cap the guest to N CPU cores (0 = keep golden default). Applied in the pre-start config PUT.")
|
|
flag.IntVar(&memoryMB, "memory", 0, "for --selftest=bring-up|provision: cap the guest RAM to N MiB (0 = keep golden default). Applied pre-start.")
|
|
flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
|
|
flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
|
|
flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
|
|
flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
|
|
flag.StringVar(&blobPath, "blob", "", "for --selftest=escrow-consume: path to the R-wrapped escrow blob file")
|
|
flag.StringVar(&expectedFP, "fingerprint", "", "for --selftest=escrow-consume: the EXPECTED key fingerprint (the gate target)")
|
|
flag.StringVar(&keyDest, "keydest", "", "for --selftest=escrow-consume: where to install the recovered key (0600)")
|
|
flag.BoolVar(&installWGKey, "install-wg-key", false, "for --selftest=identity-consume: ALSO install the recovered wg_private_key into wgtunnel's key file (S5 DR; create-only, refuses to overwrite)")
|
|
flag.StringVar(&idBundlePath, "identity-bundle", "", "for --selftest=escrow-create: a 0600 JSON file {tunnel_token,pbs_token} to ALSO escrow under R (10D)")
|
|
flag.StringVar(&directivePath, "directive", "", "for --selftest=escrow-create: a JSON file with the non-secret DR directive (pbs repo/ns, expected fingerprint, tunnel id)")
|
|
flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id — the hub config-pull target, baked into the guest's bootstrap")
|
|
flag.StringVar(&hubPassword, "hub-password", "", "for --selftest=provision: the customer's hub RETRIEVAL PASSPHRASE (SECRET) — baked into bootstrap.json so the controller pulls its config (and the customer-scoped hub key) from the hub. The customer must already exist in the hub.")
|
|
flag.StringVar(&swapImage, "image", "", "for --selftest=controller-swap: the target controller image ref (gitea.dooplex.hu/admin/felhom-controller:<semver>) — must already be pulled in the guest")
|
|
flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)")
|
|
flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)")
|
|
flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)")
|
|
flag.StringVar(&outputMode, "output", "text", "for --selftest=escrow-create: `text` (human, default — unchanged) | `json` (one machine-readable JSON object on stdout carrying the recovery code; every human line to stderr; the controller-driven ceremony's parse surface)")
|
|
flag.BoolVar(&showVersion, "version", false, "print version and exit")
|
|
flag.Parse()
|
|
|
|
if showVersion {
|
|
fmt.Println("felhom-agent", version)
|
|
return
|
|
}
|
|
|
|
cfg, err := config.Load(cfgPath)
|
|
if err != nil {
|
|
// A missing default config file is fine if env provides the values; only a
|
|
// present-but-unreadable/invalid file is fatal here.
|
|
if !(os.IsNotExist(errors.Unwrap(err)) && cfgPath == flag.Lookup("config").DefValue) {
|
|
fmt.Fprintln(os.Stderr, "config error:", err)
|
|
os.Exit(2)
|
|
}
|
|
cfg = config.Default()
|
|
}
|
|
// logRing is the always-DEBUG capture ring (v0.83.0 observability): served by the
|
|
// local API's GET /debug/logs and the heartbeat log-pull. Selftests ignore it.
|
|
logger, logRing := applog.New(cfg.LogLevel)
|
|
|
|
switch selftest.mode {
|
|
case "":
|
|
os.Exit(runDaemon(cfg, logger, logRing))
|
|
case "read":
|
|
os.Exit(runSelftestRead(context.Background(), cfg, logger))
|
|
case "task":
|
|
os.Exit(runSelftestTask(context.Background(), cfg, logger, vmid))
|
|
case "hub":
|
|
os.Exit(runSelftestHub(context.Background(), cfg, logger))
|
|
case "storage":
|
|
os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
|
|
case "backup":
|
|
os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
|
|
case "restore-test":
|
|
os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
|
|
case "pbs-verify":
|
|
os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
|
|
case "lanresolver":
|
|
os.Exit(runSelftestLANResolver(context.Background(), cfg, logger, vmid))
|
|
case "wgtunnel":
|
|
os.Exit(runSelftestWGTunnel(context.Background(), cfg, logger))
|
|
case "bring-up":
|
|
os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep,
|
|
bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
|
|
SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB}))
|
|
case "provision":
|
|
os.Exit(runSelftestProvision(context.Background(), cfg, logger, provisionArgs{
|
|
archive: archive, vmid: vmid, hostname: hostname,
|
|
customerID: custID, hubPassword: hubPassword,
|
|
sizing: bringUpSizing{RootfsGrowGB: rootfsGrow, DataVolGrowGB: dataVolGrow, DataVolMount: dataVolMount,
|
|
SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB},
|
|
}))
|
|
case "escrow-create":
|
|
os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath, outputMode))
|
|
case "escrow-consume":
|
|
os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest))
|
|
case "identity-consume":
|
|
os.Exit(runSelftestIdentityConsume(context.Background(), cfg, logger, blobPath, keyDest, installWGKey))
|
|
case "controller-swap":
|
|
os.Exit(runSelftestControllerSwap(context.Background(), cfg, logger, vmid, swapImage))
|
|
}
|
|
}
|
|
|
|
// runSelftestWGTunnel is the supervised single-shot bring-up (S3): keygen/register (marker-
|
|
// gated), one desired-state fetch, one apply, then the status stanza. Runs regardless of
|
|
// wg_tunnel.enabled (the operator invoked it deliberately) but uses the config's state
|
|
// dir/runner exactly as the daemon would.
|
|
func runSelftestWGTunnel(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
fmt.Printf("=== felhom-agent %s selftest=wgtunnel ===\n", version)
|
|
wt := cfg.WGTunnel.WithDefaults()
|
|
client, err := hub.NewClient(cfg.Hub, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] hub client:", err)
|
|
return 1
|
|
}
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
mgr := wgtunnel.NewManager(runner, client, wt.StateDir, logger)
|
|
|
|
// One registration/adopt pass with no desired data, then one fetch + apply.
|
|
mgr.Apply(ctx, false, nil)
|
|
var fetched bool
|
|
var block *hub.WireWireguard
|
|
if resp, err := client.FetchDesiredState(ctx); err != nil {
|
|
fmt.Fprintln(os.Stderr, " [WARN] desired-state fetch failed (apply skipped):", err)
|
|
} else {
|
|
fetched = true
|
|
block = resp.DesiredState.Wireguard
|
|
fmt.Printf(" desired-state generation=%d wireguard-block=%v\n", resp.Generation, block != nil)
|
|
}
|
|
mgr.Apply(ctx, fetched, block)
|
|
|
|
st := mgr.Status(ctx)
|
|
fmt.Printf(" status: pubkey=%s registered=%v active=%v assigned_ip=%s", st.Pubkey, st.Registered, st.Active, st.AssignedIP)
|
|
if st.LastHandshakeAgeS != nil {
|
|
fmt.Printf(" handshake_age_s=%d", *st.LastHandshakeAgeS)
|
|
}
|
|
fmt.Println()
|
|
if st.Registered && (block == nil || st.Active) {
|
|
fmt.Println(" [OK] wgtunnel selftest complete")
|
|
return 0
|
|
}
|
|
if !st.Registered {
|
|
fmt.Println(" [WARN] not registered (hub unreachable or endpoint unset) — see log above")
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// newProxmoxClient builds the read-path proxmox client from config.
|
|
func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) {
|
|
return proxmox.NewClient(proxmox.Config{
|
|
Endpoint: cfg.Proxmox.Endpoint,
|
|
Node: cfg.Proxmox.Node,
|
|
Token: cfg.Proxmox.Token,
|
|
TLS: proxmox.TLSConfig{
|
|
CAFile: cfg.Proxmox.TLS.CAFile,
|
|
Fingerprint: cfg.Proxmox.TLS.Fingerprint,
|
|
InsecureSkipVerify: cfg.Proxmox.TLS.InsecureSkipVerify,
|
|
},
|
|
})
|
|
}
|
|
|
|
// newHostOps builds the privileged storage surface (slice 5 Phase B) from config. It shells
|
|
// out via the same fenced Runner the proxmox layer uses (sudo -n, arg vectors, no shell);
|
|
// every argument is validated in internal/storage before any command is built. A
|
|
// missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a
|
|
// crash.
|
|
// Returns the concrete *storage.SudoHostOps (not the HostOps interface) so callers that need the
|
|
// methods outside that lean interface — the host-reboot mount re-assert and the Part-A1 network-mount
|
|
// surface — can reach them without a type assertion. It still satisfies storage.HostOps everywhere.
|
|
func newHostOps(cfg config.Config, logger *slog.Logger) *storage.SudoHostOps {
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
return storage.NewSudoHostOps(storage.SudoHostOpsConfig{
|
|
Runner: runner,
|
|
Bins: storage.Binaries{
|
|
Systemctl: cfg.Privileged.Systemctl,
|
|
Install: cfg.Privileged.Install,
|
|
Smartctl: cfg.Privileged.Smartctl,
|
|
Lvs: cfg.Privileged.Lvs,
|
|
},
|
|
UnitDir: cfg.Privileged.UnitDir,
|
|
StageDir: cfg.Privileged.StageDir,
|
|
Logger: logger,
|
|
})
|
|
}
|
|
|
|
// gateRemounter is the watchdog's benign re-mount response: it routes the re-mount through
|
|
// the reversibility gate (classified benign) and, if allowed, calls HostOps.EnsureMount. It
|
|
// lives here (not in internal/storage) so storage stays decoupled from reconcile — main is
|
|
// the one place that holds both.
|
|
type gateRemounter struct {
|
|
gate *reconcile.Gate
|
|
ops storage.HostOps
|
|
hostID string
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// Remount authorizes (benign) and performs a by-UUID re-mount of a returned target.
|
|
func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
|
|
dec := r.gate.Authorize(reconcile.IntentForStorageMount(r.hostID, t.Name), nil)
|
|
if !dec.Allowed {
|
|
r.logger.Warn("storage: re-mount refused by gate (unexpected for a benign mount)",
|
|
"target", t.Name, "reason", dec.Reason)
|
|
return
|
|
}
|
|
uuid := t.UUID
|
|
if uuid == "" {
|
|
uuid = strings.TrimPrefix(t.DurableID, "uuid:") // durable_id carries it for usb/local-dir
|
|
}
|
|
spec := storage.MountSpec{Name: t.Name, UUID: uuid, Where: t.MountPath}
|
|
if err := r.ops.EnsureMount(ctx, spec); err != nil {
|
|
r.logger.Error("storage: re-mount failed", "target", t.Name, "where", t.MountPath, "err", err)
|
|
return
|
|
}
|
|
r.logger.Info("storage: re-mounted returned target", "target", t.Name, "where", t.MountPath)
|
|
}
|
|
|
|
// poolReadStatus probes the felhom-pool membership read (GET /pools/felhom) the stale-lock reaper's
|
|
// ownership scoping depends on (v0.62.0, audit A1). Needs `Pool.Audit` at `/pool/felhom` — granted
|
|
// by host-install v1.9.0; on an older ACL this reports degraded and the reaper fail-safes (skips).
|
|
func poolReadStatus(ctx context.Context, px *proxmox.Client) capability.Status {
|
|
s := capability.Status{
|
|
Name: "pve:pool-read",
|
|
Feature: "stale-lock recovery scoping (pool ownership check)",
|
|
Critical: false,
|
|
Status: capability.StatusOK,
|
|
}
|
|
if px == nil {
|
|
s.Status, s.Reason = capability.StatusDegraded, "not configured"
|
|
return s
|
|
}
|
|
pctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
if _, err := px.Pool(pctx, reconcile.DefaultPool); err != nil {
|
|
s.Status, s.Reason = capability.StatusDegraded, err.Error()
|
|
}
|
|
return s
|
|
}
|
|
|
|
// logCapabilities logs the privileged-capability self-check at startup: one INFO summary, plus an
|
|
// ERROR per degraded capability naming the gated feature (so a missing grant is loud at cutover,
|
|
// not days later). Inactive (config-gated off, plumbing healthy — v0.86.0) is counted in the
|
|
// summary but never error-logged: disabled ≠ broken. It never exits — serve-degraded.
|
|
func logCapabilities(statuses []capability.Status, logger *slog.Logger) {
|
|
ok, total, degraded := capability.Summarize(statuses)
|
|
logger.Info("capabilities self-check",
|
|
"ok", ok, "total", total, "degraded", len(degraded), "inactive", total-ok-len(degraded))
|
|
for _, d := range degraded {
|
|
logger.Error("capability DEGRADED — privileged grant missing (feature impaired until fixed)",
|
|
"capability", d.Name, "feature", d.Feature, "reason", d.Reason, "critical", d.Critical)
|
|
}
|
|
}
|
|
|
|
// runDaemon is the default mode: collect a host-report and POST it to the hub on a
|
|
// loop. Requires both proxmox (to collect) and hub config.
|
|
func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: proxmox not configured:", err)
|
|
return 2
|
|
}
|
|
if err := cfg.Hub.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: hub not configured:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: proxmox client:", err)
|
|
return 1
|
|
}
|
|
client, err := hub.NewClient(cfg.Hub, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: hub client:", err)
|
|
return 1
|
|
}
|
|
hcfg := cfg.Hub.WithDefaults()
|
|
// Storage observer (slice 5): builds the report's storage_targets from Proxmox +
|
|
// non-privileged host reads, enriched with the privileged SMART/lvs reads via HostOps
|
|
// (Phase B). Wired into the collector via the StorageObserver seam (so hub does not
|
|
// import storage).
|
|
hostReader := storage.NewProcHostReader()
|
|
hostOps := newHostOps(cfg, logger)
|
|
observer := storage.NewObserver(px, hostReader, hostOps, logger)
|
|
// Backup + restore-test state store (slice 6): holds the latest backup-per-target +
|
|
// latest restore-test result; the collector reads it via the BackupReporter /
|
|
// RestoreTestReporter seams; the cadence scheduler writes it.
|
|
backupStore := backup.NewStore()
|
|
// PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes the shared
|
|
// store; the collector reads it via the PBSReporter seam. DR-recipe completion (v0.39.0): the
|
|
// collector now reads through a LiveSnapshotReporter that lists snapshots LIVE each collect
|
|
// (cheap GET, last-known-good fallback) so the host-report's pbs coord is present whenever PBS is
|
|
// reachable — even seconds after a restart, before the 6 h verify loop has run. The verify loop
|
|
// keeps Recording into the SAME store (shared last-known-good); targets are resolved once and
|
|
// shared by both. pbsTargets is hoisted so the reporter and the verify loop use one closure.
|
|
pbsStore := pbs.NewSnapshotStore()
|
|
pbsTargets := pbsTargetsFromPVE(cfg, px, logger)
|
|
pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargets, pbsStore, pbs.DefaultLiveSnapshotTimeout, logger)
|
|
collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsReporter, cfg.Hub.HostID, version, logger)
|
|
// Privileged-capability self-check (v0.44.0): probe the sudoers grants the non-root agent
|
|
// depends on. The probe runs `sudo -n -l` LITERALLY (a policy LIST, never executing the
|
|
// command), so it uses a DIRECT runner regardless of the agent's privileged mode. Probe once at
|
|
// startup (loud on any denial) and attach the snapshot to every hub report; the hub owns the
|
|
// ok→degraded alert. Serve-degraded — a missing grant never blocks startup.
|
|
capProber := capability.Prober{Runner: &proxmox.ExecRunner{Mode: proxmox.RunnerDirect}}
|
|
// DR-tier gate (v0.86.0, DR-tier-by-default): the pbsdr-* capabilities are config-gated — on a
|
|
// box whose DR tier is not configured (no descriptor ever / descriptor disabled) a HEALTHY
|
|
// probe reports "inactive (disabled by configuration)" instead of ok; broken plumbing (binary
|
|
// missing / grant denied) stays DEGRADED regardless (an un-migrated box must never look
|
|
// deliberately off). Late-bound: the pbsdr manager is constructed further down; probes run at
|
|
// report time. nil (pre-assignment) fails ACTIVE — the historical behavior.
|
|
var drConfigured func() bool
|
|
capProber.GateActive = func(gate string) bool {
|
|
if gate == capability.GatePBSDR && drConfigured != nil {
|
|
return drConfigured()
|
|
}
|
|
return true
|
|
}
|
|
// A1 (v0.62.0): compose the PVE pool-read check AROUND the sudo prober (an API read does not
|
|
// belong inside the sudo-policy probe). Non-critical: a degraded pool read means the stale-lock
|
|
// reaper fail-safes (locks stay uncleared) — visible on the hub report, no operator page.
|
|
probeAll := func(ctx context.Context) []capability.Status {
|
|
return append(capProber.Probe(ctx), poolReadStatus(ctx, px))
|
|
}
|
|
// (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot
|
|
// already carries the gated view — v0.86.0.)
|
|
collector.SetCapabilityProber(probeAll)
|
|
loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
|
|
interval := time.Duration(hcfg.PollSeconds) * time.Second
|
|
// Heartbeat log-pull (v0.83.0): when the control envelope requests it, the NEXT
|
|
// heartbeat carries the debug ring's tail (newest-kept, byte-capped in the loop).
|
|
if logRing != nil {
|
|
loop.SetLogTailSource(logRing.Lines)
|
|
}
|
|
|
|
// Desired-state provider (slice 10A): the hub-served target the reconcile engine converges
|
|
// toward. Starts EMPTY (generation 0) — reconcile is a live no-op until the hub serves intent,
|
|
// exactly like the slice-4 EmptyProvider. The desired.Syncer (wired below) fills it when the
|
|
// control envelope's generation advances. Replaces EmptyProvider in the engine.
|
|
desiredProvider := reconcile.NewCachingProvider()
|
|
// The "Down" channel sync hook: on each heartbeat, fetch desired-state when the generation
|
|
// advances. The loop calls it via the EnvelopeObserver seam (hub does not import desired).
|
|
desiredSyncer := desired.NewSyncer(client, desiredProvider, logger)
|
|
// S5: consume a host_loss restore_directive into an inspectable restore PLAN (derive + surface,
|
|
// execute nothing). The recipe is fetched on-demand (rare directive) via a fresh Collect.
|
|
desiredSyncer.AddConsumer(dr.NewConsumer(func(ctx context.Context) *hub.DRRecipeHostHalf {
|
|
r, err := collector.Collect(ctx)
|
|
if err != nil || r == nil {
|
|
return nil
|
|
}
|
|
return r.DRRecipe
|
|
}, cfg.Backup.RestoreStorage, logger))
|
|
// The signed-jobs runner (slice 10B) is wired as a SECOND envelope observer below (after the
|
|
// gate is built) — when the heartbeat flags pending signed ops, it fetches + verifies + executes.
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
logger.Info("felhom-agent daemon starting",
|
|
"version", version, "host_id", cfg.Hub.HostID, "hub_url", cfg.Hub.URL,
|
|
"interval_s", hcfg.PollSeconds) // hub key intentionally not logged
|
|
|
|
// Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue
|
|
// (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving
|
|
// until slice 10), so reconcile is a live no-op: it reads state and computes an
|
|
// empty action set each tick, mutating nothing. The daemon must run cleanly with
|
|
// no desired state and no signers configured — so a journal-open failure is logged
|
|
// and reconcile proceeds journal-less (it has nothing destructive to journal yet).
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err != nil {
|
|
logger.Warn("daemon: cannot ensure journal dir; reconcile runs without a journal", "path", jp, "err", err)
|
|
} else if j, err := reconcile.OpenJournal(jp); err != nil {
|
|
logger.Warn("daemon: cannot open op journal; reconcile runs without a journal", "path", jp, "err", err)
|
|
} else {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
// The reversibility gate (slice 4 Phase B) sits in front of every mutation. With
|
|
// no signers pinned (the common slice-4 state) the verifier is nil: benign actions
|
|
// pass, destructive intents are refused pending_signature — and reconcile only
|
|
// produces benign actions, so nothing is gated away. A misconfigured signer key is
|
|
// a security misconfig and is fatal.
|
|
verifier, nonceStore, err := buildVerifier(cfg, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: authz verifier:", err)
|
|
return 2
|
|
}
|
|
if nonceStore != nil {
|
|
defer nonceStore.Close()
|
|
}
|
|
gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
|
|
// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
|
|
// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
|
|
// known mount-backed target's device returning unmounted, dispatches a benign re-mount
|
|
// (routed through the gate as benign, then HostOps). With no removable/network storage
|
|
// it finds nothing to flag. The re-mount dispatch is off the poll path (a goroutine).
|
|
storageTrigger := make(chan struct{}, 1)
|
|
loop.SetTrigger(storageTrigger)
|
|
remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
|
|
// Drive intent store (slice 10 P3 self-heal): persisted, durable-id-keyed enroll/eject/decommission
|
|
// state. Gates the watchdog's self-heal re-mount to ENROLLED drives, and the local API records
|
|
// enroll/eject into it. Open failure → ungated legacy remount (logged) so the daemon still runs.
|
|
var intentReader storage.IntentReader
|
|
intentStateDir := "/var/lib/felhom-agent"
|
|
if cfg.Authz.NonceStorePath != "" {
|
|
intentStateDir = filepath.Dir(cfg.Authz.NonceStorePath) // share the agent's durable state dir
|
|
}
|
|
intentStore, ierr := storage.OpenIntentStore(filepath.Join(intentStateDir, "drive-intents.json"))
|
|
if ierr != nil {
|
|
logger.Warn("storage: intent store unavailable — self-heal runs UNGATED (legacy remount-any)", "err", ierr)
|
|
intentStore = nil
|
|
} else {
|
|
intentReader = intentStore
|
|
}
|
|
// F9: per-guest enrolled-bind record, replayed by ReassertGuestBinds at startup to restore a
|
|
// guest data-drive bind that a re-provision dropped (durable-id-keyed; absent/swapped drives skipped).
|
|
guestBindStore, gbErr := localapi.OpenGuestBindStore(filepath.Join(intentStateDir, "guest-binds.json"))
|
|
if gbErr != nil {
|
|
logger.Warn("storage: guest-bind store unavailable — startup bind re-assert disabled (F9)", "err", gbErr)
|
|
guestBindStore = nil
|
|
}
|
|
// F20-BUG3: persisted disk-format job — lets mkfs run detached from the request and survive an agent
|
|
// restart (RecoverFormatJob re-runs an interrupted durable-id-bound format).
|
|
formatJobStore, fjErr := localapi.OpenFormatJobStore(filepath.Join(intentStateDir, "format-job.json"))
|
|
if fjErr != nil {
|
|
logger.Warn("storage: format-job store unavailable — format restart-recovery disabled (F20-BUG3)", "err", fjErr)
|
|
formatJobStore = nil
|
|
}
|
|
// Impl-2a: source the watchdog's known-DRIVE set from the intent registry + Felhom .mount units
|
|
// (NOT PVE storages via Observe) — a registry-only drive (no PVE dir-storage) is now health-tracked.
|
|
// Real PVE storages (local/local-lvm/pbs) stay Observe-sourced for /disks + reports (observer below).
|
|
driveUnitDir := cfg.Privileged.UnitDir
|
|
if driveUnitDir == "" {
|
|
driveUnitDir = "/etc/systemd/system"
|
|
}
|
|
driveKnown := storage.NewRegistryKnownTargets(driveUnitDir, intentReader, logger)
|
|
// One-time migration: record pre-Impl-2a drives (mounted, with a Felhom unit) as enrolled so the
|
|
// registry-sourced Known() tracks them without their legacy PVE dir-storage. Idempotent.
|
|
if mounts, merr := hostReader.Mounts(); merr == nil {
|
|
storage.ReconcileExistingDrives(driveUnitDir, mounts, intentStore, logger)
|
|
} else {
|
|
logger.Warn("storage: existing-drive migration skipped — mount read failed", "err", merr)
|
|
}
|
|
watchdog := storage.NewWatchdog(storage.WatchdogOptions{
|
|
Targets: storage.NewCachingKnownTargets(driveKnown, cfg.Storage.KnownRefresh()),
|
|
Liveness: storage.NewHostLiveness(hostReader, 0),
|
|
Remounter: remounter,
|
|
Intent: intentReader, // P3: only re-mount enrolled drives
|
|
OnAbsent: func(durableID string) { // P3: clear `ejected` on physical absence (replug rule)
|
|
if intentStore != nil {
|
|
_ = intentStore.OnAbsent(durableID)
|
|
}
|
|
},
|
|
Trigger: func() {
|
|
select {
|
|
case storageTrigger <- struct{}{}:
|
|
default:
|
|
}
|
|
},
|
|
Interval: cfg.Storage.WatchdogInterval(),
|
|
Debounce: cfg.Storage.WatchdogDebounce(),
|
|
Logger: logger,
|
|
})
|
|
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px,
|
|
Queue: queue,
|
|
Journal: journal,
|
|
Provider: desiredProvider, // slice 10A: hub-served desired-state (empty until the hub serves intent)
|
|
Gate: gate,
|
|
HostID: cfg.Hub.HostID,
|
|
Logger: logger,
|
|
})
|
|
|
|
// Crash recovery (doc 03 §10): resolve any op that was in flight when the agent
|
|
// last died BEFORE issuing new mutations. For slice 6 this is load-bearing — a
|
|
// restore-test scratch guest leaked by a mid-test crash is torn down here.
|
|
engine.Recover(ctx)
|
|
|
|
// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
|
|
// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
|
|
// OR the scratch band / restore storage is misconfigured — the daemon still runs.
|
|
scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)
|
|
|
|
// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
|
|
// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
|
|
// verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs
|
|
// storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing.
|
|
pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
|
|
Targets: pbsTargets, // the same closure the live reporter uses (shared store, one resolver)
|
|
Store: pbsStore,
|
|
Cadence: cfg.Backup.PBSVerifyCadence(),
|
|
Logger: logger,
|
|
})
|
|
|
|
// Local API server (slice 8A, doc 03 §6): the per-guest authorization gate the in-guest
|
|
// controller calls over the bridge. Optional — runs only when local_api.enable + listen_addr
|
|
// are configured; a token-store or cert failure disables it WITHOUT killing the daemon (the
|
|
// host still reports/reconciles). The leaf is generated+persisted once so its pin is stable.
|
|
localServers := 0
|
|
var localTokens *localapi.TokenStore
|
|
var intentRec localapi.IntentRecorder
|
|
if intentStore != nil { // avoid a typed-nil interface (would defeat the nil check)
|
|
intentRec = intentStore
|
|
}
|
|
|
|
// Signed-jobs runner (slice 10B + P3): the consumer of the hub's signed-jobs queue. On a
|
|
// heartbeat that flags pending signed ops, it fetches each opaque blob, runs it through the gate
|
|
// (the LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time →
|
|
// durable nonce-burn) and, only on all-pass, hands the verified op to the executor chain:
|
|
// - storage_wipe → re-resolves the DURABLE device id + re-inspects (8C) before mkfs (closes
|
|
// the 8C data-bearing `pending_signature` gap);
|
|
// - decommission → records the PERMANENT decommission intent (keyed by the watchdog's storage
|
|
// durable-id), making the previously-unreachable IntentDecommissioned state reachable ONLY
|
|
// via a verified operator signature (never customer-confirmable; distinct from a safe eject).
|
|
// With no signers pinned the gate refuses every job (pending_signature) and nothing executes —
|
|
// correct. Wired as a second envelope observer alongside the desired-state syncer. The intent
|
|
// store is opened above (line ~340), so this wiring lives here (after it) rather than earlier.
|
|
wipeExec := signedjobs.NewWipeExecutor(hostOps, logger)
|
|
var decommIntent signedjobs.IntentDecommissioner
|
|
if intentStore != nil { // typed-nil guard (a nil *IntentStore in the interface would pass != nil)
|
|
decommIntent = intentStore
|
|
}
|
|
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
|
|
|
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
|
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
|
// detached restart). Wired as a third chain element. The commit-manager (below) commits a good
|
|
// update after a clean dwell; systemd + the wrapper auto-roll-back a crash-looping one. sudoRunner
|
|
// shells the wrapper verbs via `sudo -n` (same mode the rest of the privileged surface uses).
|
|
suCfg := cfg.SelfUpdate.WithDefaults()
|
|
suMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if suMode == "" {
|
|
suMode = proxmox.RunnerSudo
|
|
}
|
|
suRunner := &proxmox.ExecRunner{Mode: suMode, SudoPath: cfg.Privileged.SudoPath}
|
|
updateExec := selfupdate.NewExecutor(selfupdate.Config{
|
|
URLTemplate: suCfg.URLTemplate,
|
|
Username: suCfg.Username,
|
|
Token: suCfg.Token,
|
|
StateDir: suCfg.StateDir,
|
|
Runner: suRunner,
|
|
Logger: logger,
|
|
})
|
|
selfUpdateMgr := selfupdate.NewManager(selfupdate.ManagerConfig{
|
|
StateDir: suCfg.StateDir,
|
|
RunningVersion: version,
|
|
Dwell: time.Duration(suCfg.DwellSeconds) * time.Second,
|
|
Runner: suRunner,
|
|
Logger: logger,
|
|
})
|
|
collector.SetSelfUpdateReporter(selfUpdateMgr) // heartbeat pending-status field
|
|
|
|
// G1: management-plane health observer (break-glass visibility). Read-only + always wired — the
|
|
// dumb watchdog (configs/felhom-mgmt-watchdog.*) does the HEALING agent-independently; this only
|
|
// REPORTS /run/sshd presence + sshd reachability + any recent auto-heal so the hub can surface a
|
|
// recurring clobber before it locks the box out. Port 22 for G1 (H1 passes the felhom-sshd port).
|
|
collector.SetMgmtPlaneReporter(mgmtplane.NewReporter(mgmtplane.DefaultPrivsepDir, mgmtplane.DefaultHealMarker, mgmtplane.DefaultSshdPort))
|
|
|
|
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
|
|
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
|
|
|
// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
|
|
// the pbsdr manager is constructed further down; the closure reads drConfigured at call time
|
|
// (nil until then → preflight reports the tier not applied, which is the honest pre-wire answer).
|
|
escrowCeremonyCfg := &localapi.EscrowCeremonyConfig{
|
|
SudoPath: cfg.Privileged.SudoPath,
|
|
PBSStorageID: cfg.Escrow.PBSStorageID,
|
|
// Live-reload (v0.89.0): the pbsdr bridge seeds escrow.pbs_storage_id into agent.json on DR
|
|
// convergence; re-read it from disk at preflight time so the row flips green without a
|
|
// restart. Reading the same file the ceremony subprocess loads keeps the preflight honest.
|
|
// On a read error, fall back to the daemon-start snapshot. cfg.SourcePath == "" (all-env
|
|
// config) → nothing to re-read, the snapshot stands.
|
|
CurrentPBSStorageID: func() string {
|
|
if cfg.SourcePath == "" {
|
|
return cfg.Escrow.PBSStorageID
|
|
}
|
|
if c, err := config.Load(cfg.SourcePath); err == nil {
|
|
return c.Escrow.PBSStorageID
|
|
}
|
|
return cfg.Escrow.PBSStorageID
|
|
},
|
|
HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "",
|
|
DRConfigured: func() bool {
|
|
if drConfigured != nil {
|
|
return drConfigured()
|
|
}
|
|
return false
|
|
},
|
|
}
|
|
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
|
|
if localTokens != nil {
|
|
defer localTokens.Close()
|
|
}
|
|
|
|
// LAN split-horizon resolver: the agent manages a host-side dnsmasq answering *.<customer-domain>
|
|
// with each guest's live LAN IP (so LAN clients reach the box directly — same hostname, same real
|
|
// wildcard cert — instead of hairpinning through Cloudflare). Optional; runs only when
|
|
// lan_resolver.enable is set and a host LAN IP is known (explicit or derived from local_api).
|
|
lanServers := 0
|
|
var lanLoop *lanresolver.Loop
|
|
{
|
|
lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
|
|
if lr.Enabled() && lr.HostIP != "" {
|
|
lrMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if lrMode == "" {
|
|
lrMode = proxmox.RunnerSudo
|
|
}
|
|
lrRunner := &proxmox.ExecRunner{Mode: lrMode, SudoPath: cfg.Privileged.SudoPath}
|
|
mgr := lanresolver.NewManager(lrRunner, lr.HostIP, lr.Upstreams, logger)
|
|
lanLoop = lanresolver.NewLoop(mgr, lr.StateDir, time.Duration(lr.IntervalSeconds)*time.Second, logger)
|
|
logger.Info("lanresolver: enabled", "host_ip", lr.HostIP, "upstreams", lr.Upstreams, "interval_s", lr.IntervalSeconds)
|
|
} else if lr.Enabled() {
|
|
logger.Warn("lanresolver: enabled but no host IP (set lan_resolver.host_ip or local_api.listen_addr) — disabled")
|
|
}
|
|
}
|
|
|
|
// Offsite WG tunnel (S3, doc 06 §3.3): keygen + one-shot hub registration + wg-quick@wg-felhom
|
|
// managed from the hub-served desired-state block. DEFAULT OFF (the safety gate): a v0.64.0
|
|
// rollout to a box without explicit wg_tunnel.enabled=true is a no-op — no keygen, no
|
|
// registration, no report stanza.
|
|
wgServers := 0
|
|
var wgLoop *wgtunnel.Loop
|
|
var wgMgr *wgtunnel.Manager // hoisted for the fast-tick convergence source (nil when tunnel disabled)
|
|
var pokeListener *poke.Listener
|
|
{
|
|
wt := cfg.WGTunnel.WithDefaults()
|
|
if wt.Enabled {
|
|
wtMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if wtMode == "" {
|
|
wtMode = proxmox.RunnerSudo
|
|
}
|
|
wtRunner := &proxmox.ExecRunner{Mode: wtMode, SudoPath: cfg.Privileged.SudoPath}
|
|
wgMgr = wgtunnel.NewManager(wtRunner, client, wt.StateDir, logger)
|
|
wgMgr.SetStaleAfter(time.Duration(wt.StaleAfterSeconds) * time.Second)
|
|
wgLoop = wgtunnel.NewLoop(wgMgr, time.Duration(wt.IntervalSeconds)*time.Second, logger)
|
|
desiredSyncer.AddConsumer(wgLoop) // raw desired-state → the wireguard block
|
|
collector.SetWireguardReporter(wgLoop) // heartbeat status stanza
|
|
logger.Info("wgtunnel: enabled", "interval_s", wt.IntervalSeconds, "state_dir", wt.StateDir)
|
|
|
|
// Agent-plane immediate-sync LISTENER (Direction-2a, v0.89.0). Binds a contentless UDP
|
|
// poke socket EXCLUSIVELY to the box's WG /32 (from registered.json) and nudges the hub
|
|
// control loop's out-of-band report trigger — the SAME channel the storage watchdog
|
|
// uses (fan-in; the loop coalesces). Enabled whenever the tunnel is (WG is the only path
|
|
// a poke can arrive on); a lost poke is harmless — the 15-min cycle still reconciles.
|
|
pokeListener = poke.NewListener(
|
|
func() (netip.Addr, bool) { return wgtunnel.LoadAssignedAddr(wt.StateDir) },
|
|
func() {
|
|
select {
|
|
case storageTrigger <- struct{}{}:
|
|
default:
|
|
}
|
|
},
|
|
poke.Port, logger)
|
|
logger.Info("poke: agent-plane sync listener enabled", "port", poke.Port)
|
|
}
|
|
}
|
|
|
|
// OOB operator access (H1): the dedicated felhom-sshd instance + port-adaptive belt + health.
|
|
// DEFAULT OFF (oob.enabled) until the operator endpoint + static belt table exist. Consumes the
|
|
// SAME wireguard desired-state block (oob_peer_ip → belt, oob_operator_ssh_key → authorized_keys).
|
|
oobServers := 0
|
|
var oobLoop *felhomsshd.Loop
|
|
{
|
|
oc := cfg.OOB.WithDefaults()
|
|
if oc.Enabled {
|
|
oobMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if oobMode == "" {
|
|
oobMode = proxmox.RunnerSudo
|
|
}
|
|
oobRunner := &proxmox.ExecRunner{Mode: oobMode, SudoPath: cfg.Privileged.SudoPath}
|
|
oobMgr := felhomsshd.NewManager(oobRunner, oc.StateDir, logger)
|
|
oobBelt := felhomsshd.NewBelt(oobRunner, logger)
|
|
oobLoop = felhomsshd.NewLoop(oobMgr, oobBelt, time.Duration(oc.IntervalSeconds)*time.Second, logger)
|
|
desiredSyncer.AddConsumer(oobLoop) // raw desired-state → oob_peer_ip + operator key
|
|
collector.SetOOBReporter(oobLoop) // heartbeat oob health stanza
|
|
logger.Info("felhomsshd (OOB): enabled", "interval_s", oc.IntervalSeconds, "state_dir", oc.StateDir)
|
|
}
|
|
}
|
|
|
|
// PBS DR tier apply-bridge (slice 2): hub-driven — runs whenever the desired-state carries a
|
|
// pbs_dr descriptor (slice 1). No config gate: an absent/disabled descriptor is a no-op, so a
|
|
// v0.80.0 rollout changes nothing until the operator enables the tier on the hub. Adoption
|
|
// first (existing healthy entry → grant + mark, NO consume), verify-pin-before-consume,
|
|
// set-only re-apply, consumed-but-failed is loud (see internal/pbsdr).
|
|
var pbsdrLoop *pbsdr.Loop
|
|
{
|
|
pdMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if pdMode == "" {
|
|
pdMode = proxmox.RunnerSudo
|
|
}
|
|
pdRunner := &proxmox.ExecRunner{Mode: pdMode, SudoPath: cfg.Privileged.SudoPath}
|
|
secretDir := filepath.Dir(cfg.Backup.PBSSecretPath("x"))
|
|
pdMgr := pbsdr.NewManager(pdRunner, px, client, cfg.WGTunnel.WithDefaults().StateDir,
|
|
secretDir, cfg.SourcePath, logger)
|
|
pbsdrLoop = pbsdr.NewLoop(pdMgr, 60*time.Second, logger)
|
|
desiredSyncer.AddConsumer(pbsdrLoop) // raw desired-state → the pbs_dr block
|
|
collector.SetPBSDRReporter(pbsdrLoop)
|
|
// R-39 leg (c): route the live reporter's per-storage credential probe into the DR bridge, so
|
|
// a 401 becomes a LOUD `auth_failed` the hub self-heals instead of a Warn-and-skip. Without
|
|
// this wiring the probe seam exists but nothing consumes it — and the reporter deliberately
|
|
// skips probing when no sink is attached, so the whole leg would be silently inert.
|
|
pbsReporter.SetAuthSink(pdMgr)
|
|
// Capability gate wiring (v0.86.0): the prober's GatePBSDR now answers from the bridge
|
|
// (descriptor state, marker-backed across restarts) — see the capProber block above.
|
|
drConfigured = pdMgr.DRConfigured
|
|
logger.Info("pbsdr: bridge enabled (hub-driven; no-op until a pbs_dr descriptor arrives)")
|
|
}
|
|
// Startup capability self-check — after the pbsdr gate wiring so the logged snapshot matches
|
|
// what the first report will carry (inactive vs degraded is already resolved here).
|
|
logCapabilities(probeAll(context.Background()), logger)
|
|
|
|
// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
|
|
// verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop
|
|
// concurrently; any one returning ends the daemon (ctx cancel tears down the rest).
|
|
errc := make(chan error, 8)
|
|
go func() { errc <- engine.Run(ctx, interval) }()
|
|
go func() { errc <- loop.Run(ctx) }()
|
|
go func() { errc <- watchdog.Run(ctx) }()
|
|
go func() { errc <- scheduler.Run(ctx) }()
|
|
go func() { errc <- pbsLoop.Run(ctx) }()
|
|
go func() { errc <- pbsdrLoop.Run(ctx) }()
|
|
if pokeListener != nil {
|
|
go func() { errc <- pokeListener.Run(ctx) }() // agent-plane immediate-sync listener (v0.89.0)
|
|
}
|
|
|
|
// Fast-tick (v0.90.0, R-28): the agent-plane immediacy SECONDARY. While ANY desired-state item is
|
|
// unapplied — most importantly the pre-tunnel window a hub poke cannot reach — pulse the SAME
|
|
// out-of-band trigger the watchdog/poke use, every 30 s, and self-disarm the instant everything
|
|
// converges. Every source is a cached read (no exec/network per tick). The LOUD pbsdr states
|
|
// (consumed_failed/verify_failed) and the destructive pending_signature drift are deliberately
|
|
// EXCLUDED (§8) so a stuck-loud box never hammers.
|
|
fastTick := fasttick.New(storageTrigger, fasttick.DefaultInterval, logger,
|
|
fasttick.SourceFunc(func() (bool, string) {
|
|
if desiredProvider.Generation() == 0 {
|
|
return true, "desired-state not yet fetched"
|
|
}
|
|
return false, ""
|
|
}),
|
|
fasttick.SourceFunc(func() (bool, string) {
|
|
if res, ok := engine.LastResult(); ok && res.Planned-res.Pending > 0 {
|
|
return true, "reconcile drift not yet applied"
|
|
}
|
|
return false, ""
|
|
}),
|
|
fasttick.SourceFunc(func() (bool, string) {
|
|
if pbsdrLoop != nil {
|
|
if st := pbsdrLoop.PBSDRStatus(ctx); st != nil && st.State == "waiting_secret" {
|
|
return true, "pbsdr waiting for its consume-once secret"
|
|
}
|
|
}
|
|
return false, ""
|
|
}),
|
|
fasttick.SourceFunc(func() (bool, string) {
|
|
if wgMgr != nil {
|
|
if desired, operational := wgMgr.TunnelConvergence(); desired && !operational {
|
|
return true, "wireguard tunnel not yet operational"
|
|
}
|
|
}
|
|
return false, ""
|
|
}),
|
|
)
|
|
go func() { errc <- fastTick.Run(ctx) }()
|
|
if localSrv != nil {
|
|
localServers = 1
|
|
// Host-reboot remount fix: BEFORE binding into the guest, re-assert enrolled drive MOUNTS on the
|
|
// host. A `disabled` mount unit (left so by a prior detach) doesn't auto-mount at boot, and kernel
|
|
// re-enumeration can move the device (/dev/sdb→sdc); ReassertEnrolledMounts re-resolves each by
|
|
// filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op.
|
|
// Type-asserted (the concrete op exposes it; the interface stays lean).
|
|
mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts
|
|
if mountReasserter != nil {
|
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
|
}
|
|
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
|
// bind that a re-provision dropped — before serving, so the drive is back in the guest config
|
|
// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
|
|
localSrv.ReassertGuestBinds(ctx)
|
|
// F12 (CAMPAIGN-3, CRITICAL): before re-arming, migrate any already-installed network-storage
|
|
// units to the current template — the units installed before 0.85 carry the network-online
|
|
// ordering cycle that makes each host boot a coin flip (networking lost on one boot, the automount
|
|
// on the next). A general template-drift reconcile (content-hash compare, batched daemon-reload),
|
|
// idempotent. Runs BEFORE the reassert sweep so the re-armed triggers are the fixed units.
|
|
if mountReasserter != nil {
|
|
if n := mountReasserter.MigrateNetworkUnits(ctx); n > 0 {
|
|
logger.Info("network-storage units migrated to the current template (F12 boot-ordering fix)", "migrated", n)
|
|
}
|
|
}
|
|
// RCA fix 1 (AUDIT-nas-cwa-rca-2026-07-11): re-arm idle NAS automount triggers ONCE at startup —
|
|
// covers the host-boot ordering where guests autostarted before the agent (their fresh namespaces
|
|
// missed the triggers). Deliberately NOT in the 20 s ticker: an idle trigger is healthy and must
|
|
// not be churned; guest starts are covered by the guest-hook post-start leg.
|
|
localSrv.ReassertNetworkMounts(ctx)
|
|
// Intermediary-mount GUEST-REBOOT self-heal: re-run the reconcile periodically. A guest reboot
|
|
// (without an agent restart) leaves enrolled drives bound on the HOST but INVISIBLE in the fresh
|
|
// guest namespace (a non-recursive parent bind doesn't carry pre-existing submounts); the periodic
|
|
// reconcile detects the guest can't see the bind and re-fires propagation (force re-bind). Cheap
|
|
// when nothing changed (guest-visibility checks only). 20s ≪ the controller gate's 30s tick.
|
|
go func() {
|
|
t := time.NewTicker(20 * time.Second)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
// Re-assert mounts first (handles a USB that enumerated late after a host reboot, or a
|
|
// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
|
|
if mountReasserter != nil {
|
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
|
}
|
|
localSrv.ReassertGuestBinds(ctx)
|
|
}
|
|
}
|
|
}()
|
|
// CAMPAIGN-3 Part 6: node self-heal watchdog (appliance-gated). One heal — host networking
|
|
// (F12-class defense in depth: any boot leaving networking down is detected + remedied with the
|
|
// exact `systemctl start networking.service` the morning recovery ran by hand). On a byo host the
|
|
// check WARNs but the remedy is structurally unreachable (the Manager gates before any exec).
|
|
{
|
|
shMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if shMode == "" {
|
|
shMode = proxmox.RunnerSudo
|
|
}
|
|
shRunner := &proxmox.ExecRunner{Mode: shMode, SudoPath: cfg.Privileged.SudoPath}
|
|
systemctlBin := cfg.Privileged.Systemctl
|
|
if systemctlBin == "" {
|
|
systemctlBin = "/usr/bin/systemctl"
|
|
}
|
|
shMgr := &selfheal.Manager{
|
|
Appliance: cfg.IsAppliance(),
|
|
Interval: 60 * time.Second,
|
|
Heals: []selfheal.Heal{&selfheal.NetworkingHeal{
|
|
IsActive: selfheal.SystemctlIsActive("networking.service"),
|
|
HasRoute: selfheal.HasDefaultRoute,
|
|
Start: func(ctx context.Context) error {
|
|
_, stderr, err := shRunner.Run(ctx, systemctlBin, "start", "networking.service")
|
|
if err != nil {
|
|
return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(stderr)))
|
|
}
|
|
return nil
|
|
},
|
|
Sleep: selfheal.RealSleep,
|
|
Logger: logger,
|
|
}},
|
|
Logger: logger,
|
|
}
|
|
mode := "byo"
|
|
if cfg.IsAppliance() {
|
|
mode = "appliance"
|
|
}
|
|
logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
|
|
go shMgr.Watch(ctx)
|
|
}
|
|
// R-54: guest-network watchdog. Closes the OPEN RISK left by
|
|
// INCIDENT-guest-dhclient-killed-2026-07-20 §5 — the guest's DHCP client is started once by
|
|
// ifupdown at boot and nothing supervises it, so its death takes the box off the internet
|
|
// ~1-2 h later, when the lease expires. Host-tier by necessity: a guest with no default
|
|
// route cannot repair its own default route. Deliberately NOT part of the errc fan-out — a
|
|
// watchdog over customer guests must never be able to bring the agent down.
|
|
if gn := cfg.GuestNet.WithDefaults(); gn.Enabled() {
|
|
gnMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if gnMode == "" {
|
|
gnMode = proxmox.RunnerSudo
|
|
}
|
|
gnRunner := &proxmox.ExecRunner{Mode: gnMode, SudoPath: cfg.Privileged.SudoPath}
|
|
// The guest source is the POOL-VERIFIED one (ListLXC ∩ felhom pool, audit A1) — never a
|
|
// bare ListLXC, which under a broad token would run dhclient inside a co-tenant's guest.
|
|
gnGuests := localapi.NewStaleLockController(px, gnRunner, reconcile.DefaultPool, logger)
|
|
gnWatchdog := guestnet.New(gnRunner, gnGuests, time.Duration(gn.IntervalSeconds)*time.Second, logger)
|
|
gnWatchdog.SetDampers(
|
|
time.Duration(gn.MinHealIntervalSeconds)*time.Second,
|
|
gn.MaxHealsPerHour,
|
|
time.Duration(gn.SettleSeconds)*time.Second,
|
|
)
|
|
collector.SetGuestNetReporter(gnWatchdog)
|
|
go gnWatchdog.Watch(ctx)
|
|
} else {
|
|
logger.Info("guestnet: watchdog disabled by config (guest_net.disable)")
|
|
}
|
|
// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
|
|
localSrv.RecoverFormatJob(ctx)
|
|
// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete
|
|
// the dangling snapshot → start iff onboot). Runs BEFORE the backup loop starts, so a present
|
|
// backup lock is stale by definition (guarded by a no-vzdump-running check; fail-safe otherwise).
|
|
localSrv.RecoverStaleLockedGuests(ctx)
|
|
go func() { errc <- localSrv.Run(ctx) }()
|
|
}
|
|
if lanLoop != nil {
|
|
lanServers = 1
|
|
go func() { errc <- lanLoop.Run(ctx) }()
|
|
}
|
|
if wgLoop != nil {
|
|
wgServers = 1
|
|
go func() { errc <- wgLoop.Run(ctx) }()
|
|
}
|
|
if oobLoop != nil {
|
|
oobServers = 1
|
|
go func() { errc <- oobLoop.Run(ctx) }()
|
|
}
|
|
|
|
// TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS
|
|
// version), commit it after a clean dwell — but only now that core init is done (config parsed,
|
|
// hub loop + storage watchdog + local API all started above). Runs in its own goroutine so the
|
|
// dwell never blocks the daemon; it is NOT one of the errc siblings (it returns after commit and
|
|
// must not end the daemon). A crash before the commit → systemd + the wrapper roll back to .prev.
|
|
go selfUpdateMgr.MaybeCommit(ctx)
|
|
|
|
err = <-errc
|
|
stop() // tear down the siblings on the first exit
|
|
for i := 0; i < 4+localServers+lanServers+wgServers+oobServers; i++ { // wait for the other goroutines
|
|
<-errc
|
|
}
|
|
if err != nil && err != context.Canceled {
|
|
logger.Error("daemon: exited with error", "err", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// pbsSecretReader is the seam that reads a PBS storage's token secret (R-39 leg b). Overridable in
|
|
// tests; production is readPBSSecretViaWrapper.
|
|
var pbsSecretReader = readPBSSecretViaWrapper
|
|
|
|
// readPBSSecret reads a storage's token secret, preferring a directly-readable file and falling back
|
|
// to the root wrapper.
|
|
//
|
|
// WHY THE FALLBACK EXISTS. The agent runs NON-ROOT and writes /etc/pve/priv/storage/<id>.pw through
|
|
// the root wrapper — but /etc/pve/priv is 0700 root:www-data, so it could never read that file back.
|
|
// The old code called readTrimmed on it directly, got "permission denied" every cycle, logged a Warn
|
|
// and SKIPPED the datastore. That is why the 401 in R-39 went unnoticed for weeks: the one loop that
|
|
// could have caught it was blind by construction, not by accident.
|
|
//
|
|
// The direct read is kept first because a box configured with its own agent-owned secret dir
|
|
// (place_copies puts a 0600 felhom-agent copy there) needs no sudo at all; the wrapper is the path
|
|
// for the default PRIVDIR case.
|
|
func readPBSSecret(ctx context.Context, cfg config.Config, storageID string) (string, error) {
|
|
path := cfg.Backup.PBSSecretPath(storageID)
|
|
if secret, err := readTrimmed(path); err == nil && secret != "" {
|
|
return secret, nil
|
|
}
|
|
return pbsSecretReader(ctx, cfg, storageID)
|
|
}
|
|
|
|
// readPBSSecretViaWrapper shells the root wrapper's `read` verb. The secret arrives on STDOUT and is
|
|
// never passed through argv (sudo logs argv) and never logged.
|
|
func readPBSSecretViaWrapper(ctx context.Context, cfg config.Config, storageID string) (string, error) {
|
|
dir := filepath.Dir(cfg.Backup.PBSSecretPath(storageID))
|
|
cmd := exec.CommandContext(ctx, "sudo", "-n", pbsdr.WrapperPath, "read", storageID, dir)
|
|
var out, errb bytes.Buffer
|
|
cmd.Stdout = &out
|
|
cmd.Stderr = &errb
|
|
if err := cmd.Run(); err != nil {
|
|
// The wrapper's refusal text is safe to surface (it never echoes the value).
|
|
return "", fmt.Errorf("wrapper read %s: %w: %s", storageID, err, strings.TrimSpace(errb.String()))
|
|
}
|
|
secret := strings.TrimSpace(out.String())
|
|
if secret == "" {
|
|
return "", fmt.Errorf("wrapper read %s: empty secret", storageID)
|
|
}
|
|
return secret, nil
|
|
}
|
|
|
|
// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs
|
|
// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each
|
|
// (token id from the storage `username`, secret read from <PBSSecretDir>/<id>.pw). A storage
|
|
// whose secret/client can't be built is skipped with a warning (the loop still verifies the
|
|
// rest). The secret is read at runtime, never logged.
|
|
func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets {
|
|
return func(ctx context.Context) ([]pbs.Target, error) {
|
|
stores, err := px.ListStorage(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var targets []pbs.Target
|
|
for _, s := range stores {
|
|
if s.Type != "pbs" {
|
|
continue
|
|
}
|
|
secret, err := readPBSSecret(ctx, cfg, s.Storage)
|
|
if err != nil {
|
|
logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
|
|
continue
|
|
}
|
|
c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret, Namespace: s.Namespace})
|
|
if err != nil {
|
|
logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
|
|
continue
|
|
}
|
|
targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c, StorageID: s.Storage})
|
|
}
|
|
return targets, nil
|
|
}
|
|
}
|
|
|
|
// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
|
|
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
|
|
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
|
|
stores, err := px.ListStorage(ctx)
|
|
if err != nil {
|
|
return "local"
|
|
}
|
|
for _, s := range stores {
|
|
if s.Storage == storageID && s.Type == "pbs" {
|
|
return "pbs"
|
|
}
|
|
}
|
|
return "local"
|
|
}
|
|
|
|
// restoreTaskTimeout returns the tier-aware restore-task wait: the generous configured PBS timeout
|
|
// for a WAN (pbs-tier) restore, else 0 (→ WaitOptions' 10m default) for a local restore. S4.1: a
|
|
// too-short wait kills a WAN restore mid-flight → mid-restore teardown → leaked scratch.
|
|
func restoreTaskTimeout(cfg config.Config, tier string) time.Duration {
|
|
if tier == "pbs" {
|
|
return cfg.Backup.RestoreTestPBSRestoreTimeout()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
|
|
func readTrimmed(path string) (string, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
s := strings.TrimSpace(string(b))
|
|
if s == "" {
|
|
return "", fmt.Errorf("empty file %s", path)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
|
|
// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
|
|
// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
|
|
// machinery still works on-demand via --selftest=restore-test.
|
|
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, logger *slog.Logger) *backup.Scheduler {
|
|
cadence := cfg.Backup.RestoreTestCadence()
|
|
if cadence > 0 {
|
|
if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
|
|
logger.Warn("daemon: restore-test cadence disabled (config invalid)", "err", err)
|
|
cadence = 0
|
|
}
|
|
}
|
|
min, max := cfg.Backup.ScratchBand()
|
|
target := cfg.Backup.BackupTarget()
|
|
runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
|
|
return backup.NewScheduler(backup.SchedulerOptions{
|
|
Runner: engine,
|
|
Pick: runner.PickRestoreCandidate,
|
|
Store: store,
|
|
Spec: func() reconcile.RestoreTestSpec {
|
|
tier := storageTier(context.Background(), px, target)
|
|
return reconcile.RestoreTestSpec{
|
|
RestoreStorage: cfg.Backup.RestoreStorage,
|
|
ScratchMin: min,
|
|
ScratchMax: max,
|
|
SourceTier: tier,
|
|
RestoreTaskTimeout: restoreTaskTimeout(cfg, tier),
|
|
}
|
|
}(),
|
|
Cadence: cadence,
|
|
Logger: logger,
|
|
})
|
|
}
|
|
|
|
// buildLocalAPIServer constructs the per-guest local-API server (slice 8A, doc 03 §6) when
|
|
// configured. It opens the durable hashed token store and ensures the persisted self-signed
|
|
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
|
|
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
|
|
// fixed. The opened token store is returned via outTokens so the caller can Close it.
|
|
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
|
|
if !cfg.LocalAPI.Enabled() {
|
|
return nil
|
|
}
|
|
if err := cfg.LocalAPI.Validate(); err != nil {
|
|
logger.Warn("daemon: local-api disabled (config invalid)", "err", err)
|
|
return nil
|
|
}
|
|
tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (token store)", "err", err)
|
|
return nil
|
|
}
|
|
*outTokens = tokens
|
|
host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
|
|
cert, fp, generated, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (leaf cert)", "err", err)
|
|
return nil
|
|
}
|
|
if generated {
|
|
// B.1: a freshly-minted leaf invalidates every already-issued bootstrap pin. LOUD so an
|
|
// accidental regeneration (e.g. a state-dir move during a reinstall) is caught immediately.
|
|
logger.Warn("local-api leaf REGENERATED — any previously issued bootstrap pins are now INVALID; controllers will fail the pin check until re-bootstrapped (or restore the prior leaf)",
|
|
"fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
|
} else {
|
|
logger.Info("local-api leaf LOADED", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
|
}
|
|
// v0.48.0: ride the served leaf fp on every host report so the hub can detect a re-key fleet-wide.
|
|
collector.SetLeafFingerprint(fp)
|
|
runner := backup.NewBackupRunner(px, cfg.Backup.BackupTarget(), "", "felhom local-api", cfg.Backup.PruneBackupsSpec(), logger)
|
|
// Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown
|
|
// (same fenced ExecRunner the host-storage + provision back-half use).
|
|
gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if gaMode == "" {
|
|
gaMode = proxmox.RunnerSudo
|
|
}
|
|
guestBinder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
|
|
srv, err := localapi.NewServer(localapi.Options{
|
|
ListenAddr: cfg.LocalAPI.ListenAddr,
|
|
Cert: cert,
|
|
AgentVersion: version, // v0.82.0: the X-Felhom-Agent-Version capability channel
|
|
Guests: px,
|
|
Backups: runner,
|
|
Store: store,
|
|
Storage: observer,
|
|
DriveTargets: driveTargets, // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages)
|
|
Smart: storage.NewSmartReader(hostOps), // v0.95.0 Fix B: SMART for the union-path drives
|
|
HostReader: storage.NewProcHostReader(), // Impl-2b: durableIDForMount raw-mount fallback + role gate
|
|
Tokens: tokens,
|
|
BackupCadence: cfg.Backup.BackupCadence(),
|
|
// Disk management (slice 8C): the privileged host surface + the data-bearing wipe gate.
|
|
Disks: hostOps,
|
|
DiskGate: storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID},
|
|
Guests2: px,
|
|
GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest
|
|
Memory: px, // v0.90.0 R-24: guest RAM resize (SetConfig live cgroup apply)
|
|
// Network storage (NAS) — Part A1: the privileged host network-mount surface (NFS/SMB automount).
|
|
NetStorage: hostOps,
|
|
SmbCredsDir: cfg.Privileged.SmbCredsDir,
|
|
ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap
|
|
// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
|
|
// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
|
|
// A1 (v0.62.0): the scan is restricted to felhom-pool members (ownership proven, not assumed).
|
|
StaleLock: localapi.NewStaleLockController(px, &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, reconcile.DefaultPool, logger),
|
|
Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal
|
|
GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert
|
|
FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery
|
|
|
|
// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
|
|
// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
|
|
HostMetrics: collector,
|
|
HostID: cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op
|
|
LogRing: logRing, // v0.83.0: GET /debug/logs — the always-DEBUG capture ring
|
|
// Controller-driven escrow ceremony (v0.88.0): the customer wizard's agent half.
|
|
EscrowCeremony: escrowCeremony,
|
|
Logger: logger,
|
|
})
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (server build)", "err", err)
|
|
return nil
|
|
}
|
|
return srv
|
|
}
|
|
|
|
// storageGateAdapter bridges the local-API disk-format path to the reversibility gate, TIERED by the
|
|
// agent's authoritative device-role verdict: a USER-DATA data-bearing wipe is customer-confirmable
|
|
// (durable-id-bound, no signature); a SYSTEM/BACKUP wipe stays operator-signature (unsigned →
|
|
// pending_signature; the signed completion runs via the signed-jobs runner).
|
|
type storageGateAdapter struct {
|
|
gate *reconcile.Gate
|
|
hostID string
|
|
}
|
|
|
|
func (a storageGateAdapter) AuthorizeWipe(req localapi.WipeRequest) localapi.WipeDecision {
|
|
dec := a.gate.AuthorizeStorageWipe(reconcile.StorageWipeAuthz{
|
|
HostID: a.hostID,
|
|
Role: req.Role,
|
|
DeviceDurableID: req.DeviceDurableID,
|
|
Confirmed: req.Confirmed,
|
|
ConfirmDurableID: req.ConfirmDurableID,
|
|
}, nil) // no operator signature on the inline path → system/backup is pending_signature
|
|
tier := "destructive"
|
|
if dec.Disposition == reconcile.CustomerConfirmable {
|
|
tier = "customer_confirmable"
|
|
}
|
|
return localapi.WipeDecision{
|
|
Allowed: dec.Allowed,
|
|
Tier: tier,
|
|
Reason: string(dec.Reason),
|
|
NeedsConfirmation: dec.Reason == reconcile.ReasonPendingConfirmation,
|
|
}
|
|
}
|
|
|
|
// reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
|
|
// configured nonce store (both are durable agent state), falling back to the standard
|
|
// host state dir when the nonce store is unset.
|
|
func reconcileJournalPath(cfg config.Config) string {
|
|
if p := cfg.Authz.NonceStorePath; p != "" {
|
|
return filepath.Join(filepath.Dir(p), "journal.log")
|
|
}
|
|
return "/var/lib/felhom-agent/journal.log"
|
|
}
|
|
|
|
// buildVerifier constructs the operator-signed-op verifier from config. With no signers
|
|
// pinned it returns (nil, nil, nil) — the gate then refuses any destructive intent as
|
|
// pending_signature (and reconcile serves none). With signers it requires a durable
|
|
// nonce-store path (anti-replay must survive restarts) and parses every pinned key —
|
|
// a bad key or missing path is a fatal security misconfig.
|
|
func buildVerifier(cfg config.Config, logger *slog.Logger) (reconcile.OpVerifier, *authz.FileNonceStore, error) {
|
|
if len(cfg.Authz.Signers) == 0 {
|
|
logger.Info("daemon: no operator signers pinned; destructive ops will be refused pending_signature")
|
|
return nil, nil, nil
|
|
}
|
|
if cfg.Authz.NonceStorePath == "" {
|
|
return nil, nil, fmt.Errorf("authz.nonce_store_path is required when signers are configured")
|
|
}
|
|
signers := make([]authz.AllowedSigner, 0, len(cfg.Authz.Signers))
|
|
for _, s := range cfg.Authz.Signers {
|
|
as, err := authz.NewAllowedSigner(s.KeyID, authz.KeyRole(s.Role), s.PublicKey)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("pinned signer %q: %w", s.KeyID, err)
|
|
}
|
|
signers = append(signers, as)
|
|
}
|
|
store, err := authz.OpenFileNonceStore(cfg.Authz.NonceStorePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("nonce store: %w", err)
|
|
}
|
|
logger.Info("daemon: operator signers pinned", "count", len(signers))
|
|
return authz.New(signers, store, cfg.Hub.HostID), store, nil
|
|
}
|
|
|
|
// runSelftestControllerSwap exercises the agentic controller-update swap primitive directly (Phase 1):
|
|
// it swaps guest -vmid's controller to -image, verifies it comes up healthy, and ROLLS BACK if not.
|
|
// The target image must already be pulled in the guest (the controller pre-pulls it in the real flow).
|
|
func runSelftestControllerSwap(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int, image string) int {
|
|
if vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=controller-swap: -vmid is required")
|
|
return 1
|
|
}
|
|
if !localapi.ValidControllerImage(image) {
|
|
fmt.Fprintln(os.Stderr, "selftest=controller-swap: -image must be gitea.dooplex.hu/admin/felhom-controller:<semver>")
|
|
return 1
|
|
}
|
|
gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if gaMode == "" {
|
|
gaMode = proxmox.RunnerSudo
|
|
}
|
|
binder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger)
|
|
swapper := localapi.NewControllerSwapper(binder, "/var/lib/felhom-agent", logger)
|
|
|
|
cur, _ := swapper.CurrentImage(ctx, vmid)
|
|
fmt.Printf("=== felhom-agent %s selftest=controller-swap (vmid=%d) ===\n", version, vmid)
|
|
fmt.Printf(" current image: %s\n target image: %s\n", cur, image)
|
|
|
|
st := swapper.Swap(ctx, vmid, image)
|
|
fmt.Printf(" --- result ---\n state=%s current=%s previous=%s\n", st.State, st.Current, st.Previous)
|
|
if st.Error != "" {
|
|
fmt.Printf(" error: %s\n", st.Error)
|
|
}
|
|
if st.State == "done" {
|
|
fmt.Println("=== selftest=controller-swap OK (swapped + healthy) ===")
|
|
return 0
|
|
}
|
|
fmt.Println("=== selftest=controller-swap FAILED (rolled back if previous existed) ===")
|
|
return 1
|
|
}
|
|
|
|
// runSelftestHub validates hub config, does ONE collect + report, and prints the
|
|
// report it would send plus the envelope it got back.
|
|
func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if err := cfg.Hub.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: hub not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
client, err := hub.NewClient(cfg.Hub, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: hub client:", err)
|
|
return 1
|
|
}
|
|
observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
|
|
// Wire the LIVE PBS reporter here too (v0.39.0): selftest=hub is a separate one-shot process — no
|
|
// verify loop runs — so a nil reporter previously yielded pbs_snapshots:[] and an absent dr_recipe
|
|
// pbs coord. The live reporter lists snapshots directly (fresh store, last-known-good fallback) so
|
|
// the selftest reflects exactly what a freshly-restarted daemon's first collect emits.
|
|
pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargetsFromPVE(cfg, px, logger), pbs.NewSnapshotStore(), pbs.DefaultLiveSnapshotTimeout, logger)
|
|
collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, pbsReporter, cfg.Hub.HostID, version, logger)
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=hub (host_id=%s url=%s) ===\n", version, cfg.Hub.HostID, cfg.Hub.URL)
|
|
report, err := collector.Collect(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] collect:", err)
|
|
return 1
|
|
}
|
|
if b, e := json.MarshalIndent(report, " ", " "); e == nil {
|
|
fmt.Println(" --- report it would send ---")
|
|
fmt.Println(" " + string(b))
|
|
}
|
|
env, err := client.Report(ctx, report)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] report:", err)
|
|
return 1
|
|
}
|
|
if b, e := json.MarshalIndent(env, " ", " "); e == nil {
|
|
fmt.Println(" --- control envelope received ---")
|
|
fmt.Println(" " + string(b))
|
|
}
|
|
fmt.Println("=== selftest=hub OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestStorage is the live storage harness (slice 5 Phase B, for the USB runbook).
|
|
// It needs Proxmox config only (NO hub) so it runs standalone on the Proxmox host:
|
|
// - observe pass: print the full StorageTarget table incl. the privileged SMART summary
|
|
// and thin-pool data+metadata fill.
|
|
// - -watch D: run the watchdog verbose for D with the re-mount response LIVE, so the
|
|
// operator can physically cycle a drive and watch detect → report → re-mount.
|
|
func runSelftestStorage(ctx context.Context, cfg config.Config, logger *slog.Logger, watch time.Duration) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
hostReader := storage.NewProcHostReader()
|
|
hostOps := newHostOps(cfg, logger)
|
|
observer := storage.NewObserver(px, hostReader, hostOps, logger)
|
|
|
|
octx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
|
fmt.Printf("=== felhom-agent %s selftest=storage ===\n", version)
|
|
targets, err := observer.Observe(octx)
|
|
cancel()
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] observe:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" observed %d storage target(s):\n", len(targets))
|
|
for _, t := range targets {
|
|
fmt.Printf(" - %-12s type=%-9s state=%-12s reach=%-5v class=%-4s durable=%s\n",
|
|
t.Name, t.Type, t.State, t.Reachable, t.ClassHint, t.DurableID)
|
|
fmt.Printf(" usage %s/%s (%.0f%%) mount=%q dev=%q\n",
|
|
gib(t.UsedBytes), gib(t.TotalBytes), t.UsedFraction*100, t.MountPath, t.BackingDevice)
|
|
fmt.Printf(" smart: health=%s%s\n", t.Smart.Health, smartCounters(t.Smart))
|
|
if t.ThinPool != nil {
|
|
meta := "n/a"
|
|
if t.ThinPool.MetadataUsedFraction != nil {
|
|
meta = fmt.Sprintf("%.1f%%", *t.ThinPool.MetadataUsedFraction*100)
|
|
}
|
|
fmt.Printf(" thin-pool: data=%.1f%% metadata=%s\n", t.ThinPool.DataUsedFraction*100, meta)
|
|
}
|
|
}
|
|
|
|
if watch <= 0 {
|
|
fmt.Println("=== selftest=storage OK (observe pass; pass -watch D for the live watchdog) ===")
|
|
return 0
|
|
}
|
|
|
|
// Live watchdog window: re-mount response active. Cycle a drive and watch the logs.
|
|
fmt.Printf(" --- watching for %s (cycle a drive now; detect → report → re-mount) ---\n", watch)
|
|
wctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
wctx, cancel2 := context.WithTimeout(wctx, watch)
|
|
defer cancel2()
|
|
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
|
|
wd := storage.NewWatchdog(storage.WatchdogOptions{
|
|
Targets: storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
|
|
Liveness: storage.NewHostLiveness(hostReader, 0),
|
|
Remounter: remounter,
|
|
Trigger: func() { logger.Info("storage: (selftest) would send out-of-band host-report now") },
|
|
Interval: cfg.Storage.WatchdogInterval(),
|
|
Debounce: cfg.Storage.WatchdogDebounce(),
|
|
Logger: logger,
|
|
})
|
|
_ = wd.Run(wctx)
|
|
fmt.Println("=== selftest=storage watch window ended ===")
|
|
return 0
|
|
}
|
|
|
|
// smartCounters renders the non-nil SMART counters compactly for the selftest table.
|
|
func smartCounters(s hub.SmartSummary) string {
|
|
var parts []string
|
|
add := func(name string, v *int) {
|
|
if v != nil {
|
|
parts = append(parts, fmt.Sprintf("%s=%d", name, *v))
|
|
}
|
|
}
|
|
add("temp", s.TemperatureC)
|
|
add("poh", s.PowerOnHours)
|
|
add("realloc", s.ReallocatedSectors)
|
|
add("pending", s.PendingSectors)
|
|
add("offline_unc", s.OfflineUncorrectable)
|
|
add("crit_warn", s.CriticalWarning)
|
|
add("media_err", s.MediaErrors)
|
|
add("pct_used", s.PercentageUsed)
|
|
if len(parts) == 0 {
|
|
return ""
|
|
}
|
|
return " " + strings.Join(parts, " ")
|
|
}
|
|
|
|
// runSelftestBackup runs one vzdump of -vmid to the configured local backup target and
|
|
// prints the resulting Backup record. Standalone (no hub). Live + benign.
|
|
func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if vmid == 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
|
|
return 2
|
|
}
|
|
target := cfg.Backup.BackupTarget()
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, target)
|
|
runner := backup.NewBackupRunner(px, target, "", "felhom selftest", "", logger)
|
|
rec, err := runner.Backup(ctx, vmid)
|
|
printJSON("backup record", rec)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] backup:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=backup OK (crash-consistent=%v, archive=%s) ===\n", rec.CrashConsistent, rec.Archive)
|
|
return 0
|
|
}
|
|
|
|
// runSelftestRestoreTest runs one self-restore-test (restore → net-link-down → boot → verify
|
|
// running → teardown) of -archive (or the newest backup on the local target) into a scratch
|
|
// guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
|
|
// crashed test is reaped before this run.
|
|
func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=restore-test:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
// Build a journaled engine so a leaked scratch from a prior crash is recoverable.
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
})
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=restore-test ===\n", version)
|
|
fmt.Println(" --- recover: reaping any leaked scratch from a prior crashed test ---")
|
|
rec := engine.Recover(ctx)
|
|
fmt.Printf(" recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)
|
|
|
|
target := cfg.Backup.BackupTarget()
|
|
if archive == "" {
|
|
runner := backup.NewBackupRunner(px, target, "", "", "", logger)
|
|
archive, err = runner.PickRestoreCandidate(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] pick backup:", err)
|
|
return 1
|
|
}
|
|
if archive == "" {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", target, "(run --selftest=backup first)")
|
|
return 1
|
|
}
|
|
}
|
|
min, max := cfg.Backup.ScratchBand()
|
|
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
|
rtTier := storageTier(ctx, px, target)
|
|
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
|
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
|
ScratchMin: min, ScratchMax: max, SourceTier: rtTier,
|
|
RestoreTaskTimeout: restoreTaskTimeout(cfg, rtTier),
|
|
})
|
|
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
|
if res.Skipped {
|
|
fmt.Println("=== selftest=restore-test SKIPPED (no free scratch VMID in band) ===")
|
|
return 0
|
|
}
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] restore-test (scratch %d): %v\n", res.ScratchVMID, res.Err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=restore-test OK (scratch %d restored+booted+verified+torn-down in %s) ===\n", res.ScratchVMID, res.Duration.Round(time.Second))
|
|
return 0
|
|
}
|
|
|
|
// runSelftestBringUp runs the REAL journaled bring-up job (slice 7) on-demand: restore →
|
|
// reset identity → size → attach mounts → start link-up, then tears the guest down by default
|
|
// (a selftest must not leave a guest running) unless -keep. -mode picks provision (golden, fresh
|
|
// identity) or dr (customer backup, preserve continuity). It first Recovers, so a leaked guest
|
|
// from a prior crashed bring-up is reaped before this run.
|
|
// bringUpSizing carries the OS-rootfs / Docker-data sizing knobs from the CLI into a bring-up.
|
|
// Defaults (0/"") keep the golden's baked sizes; the provisioning spec sources these per-customer
|
|
// (flags now; the slice-10 hub storage manifest later — see bringup.go GuestMount comment).
|
|
type bringUpSizing struct {
|
|
RootfsGrowGB int
|
|
DataVolGrowGB int
|
|
DataVolMount string
|
|
SysDataGrowGB int
|
|
SysDataMount string
|
|
Cores int // 0 = keep golden default (CPU-core cap applied pre-start)
|
|
MemoryMB int // 0 = keep golden default (RAM cap in MiB, applied pre-start)
|
|
}
|
|
|
|
func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Logger, mode, archive string, vmid int, hostname string, keep bool, sizing bringUpSizing) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
var bmode reconcile.BringUpMode
|
|
switch mode {
|
|
case "provision":
|
|
bmode = reconcile.ModeProvision
|
|
case "dr":
|
|
bmode = reconcile.ModeDRGuestLoss
|
|
default:
|
|
fmt.Fprintf(os.Stderr, "selftest=bring-up: -mode must be provision|dr (got %q)\n", mode)
|
|
return 2
|
|
}
|
|
if archive == "" || vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=bring-up requires -archive <volid> and -vmid <N>")
|
|
return 2
|
|
}
|
|
if cfg.Backup.RestoreStorage == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=bring-up requires backup.restore_storage in config")
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
// GL-5: DR bring-up swaps the structural binds (mp8/mp9) via root pct ops — wire the same
|
|
// Runner shape the provision back-half uses. Provision mode never touches it.
|
|
rMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if rMode == "" {
|
|
rMode = proxmox.RunnerSudo
|
|
}
|
|
hostRunner := &proxmox.ExecRunner{Mode: rMode, SudoPath: cfg.Privileged.SudoPath}
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
HostRunner: hostRunner,
|
|
})
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
|
|
fmt.Println(" --- recover: reaping any half-built guest from a prior crashed bring-up ---")
|
|
rec := engine.Recover(ctx)
|
|
fmt.Printf(" recover: examined=%d bring_up_rolled_back=%d bring_up_clean=%d scratch_destroyed=%d\n",
|
|
rec.Examined, rec.BringUpRolledBack, rec.BringUpClean, rec.ScratchDestroyed)
|
|
|
|
spec := reconcile.BringUpSpec{
|
|
Mode: bmode, Archive: archive, VMID: vmid, RestoreStorage: cfg.Backup.RestoreStorage,
|
|
Hostname: hostname, KeepMAC: bmode == reconcile.ModeDRGuestLoss,
|
|
Pool: reconcile.DefaultPool, // restore into the felhom pool (both provision + DR); required under the scoped token
|
|
Cores: sizing.Cores, MemoryMB: sizing.MemoryMB,
|
|
RootfsGrowGB: sizing.RootfsGrowGB, DataVolGrowGB: sizing.DataVolGrowGB, DataVolMount: sizing.DataVolMount,
|
|
SysDataGrowGB: sizing.SysDataGrowGB, SysDataMount: sizing.SysDataMount,
|
|
}
|
|
fmt.Printf(" bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
|
|
res := engine.RunBringUp(ctx, spec)
|
|
printJSON("bring-up record", res)
|
|
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] bring-up (vmid %d): %v\n", vmid, res.Err)
|
|
// On failure the job already compensating-rolled-back; nothing to tear down.
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
|
|
if len(res.StartWarnings) > 0 {
|
|
fmt.Printf(" start warnings (recognized=%v): %v\n", res.WarningsRecognized, res.StartWarnings)
|
|
}
|
|
|
|
if keep {
|
|
fmt.Printf("=== selftest=bring-up OK — guest %d KEPT (-keep) ===\n", vmid)
|
|
return 0
|
|
}
|
|
// A selftest must not leave a guest running. Tear it down (out-of-band, like the spike).
|
|
fmt.Printf(" --- teardown: destroying selftest guest %d ---\n", vmid)
|
|
upid, err := px.DestroyLXC(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " [WARN] teardown destroy: %v (destroy vmid %d manually)\n", err, vmid)
|
|
return 1
|
|
}
|
|
if upid != "" {
|
|
if _, err := px.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
|
fmt.Fprintf(os.Stderr, " [WARN] teardown destroy task: %v (check vmid %d)\n", err, vmid)
|
|
return 1
|
|
}
|
|
}
|
|
// GL-5: the DR bind swap created <stateDir>/guests/<vmid>/bootstrap for this SCRATCH vmid —
|
|
// remove the agent-owned per-guest dir with the guest (never a real drive's bind source; a
|
|
// scratch vmid has no other state). Best-effort.
|
|
if bmode == reconcile.ModeDRGuestLoss {
|
|
if err := os.RemoveAll(filepath.Join("/var/lib/felhom-agent/guests", strconv.Itoa(vmid))); err != nil {
|
|
fmt.Fprintf(os.Stderr, " [WARN] scratch mp9 host dir cleanup: %v\n", err)
|
|
}
|
|
}
|
|
fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
|
|
return 0
|
|
}
|
|
|
|
// provisionArgs bundles the --selftest=provision inputs.
|
|
type provisionArgs struct {
|
|
archive string
|
|
vmid int
|
|
hostname string
|
|
customerID string // baked into bootstrap (the hub config-pull target)
|
|
hubPassword string // the customer's hub retrieval passphrase (SECRET) — baked into bootstrap
|
|
sizing bringUpSizing // OS-rootfs / Docker-data sizing for the bring-up half
|
|
}
|
|
|
|
// sanitizeHostname makes s a DNS-safe LXC hostname (RFC 1123 label-ish): lowercase, any run of
|
|
// invalid characters collapses to a single '-', leading/trailing '-' stripped, capped at 63 chars.
|
|
// Returns "" if nothing usable remains (caller then sets no hostname). PVE itself validates, but a
|
|
// customer id can legitimately contain characters (e.g. '_' or spaces) that a hostname cannot.
|
|
func sanitizeHostname(s string) string {
|
|
s = strings.ToLower(strings.TrimSpace(s))
|
|
var b strings.Builder
|
|
prevDash := false
|
|
for _, r := range s {
|
|
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
|
b.WriteRune(r)
|
|
prevDash = false
|
|
} else if !prevDash && b.Len() > 0 {
|
|
b.WriteByte('-')
|
|
prevDash = true
|
|
}
|
|
}
|
|
out := strings.Trim(b.String(), "-")
|
|
if len(out) > 63 {
|
|
out = strings.Trim(out[:63], "-")
|
|
}
|
|
return out
|
|
}
|
|
|
|
// runSelftestProvision runs the FULL slice-8A provisioning chain on-demand: the slice-7 bring-up
|
|
// FRONT half (provision mode, golden) + the slice-8A BACK half (mint per-guest token → render the
|
|
// stable bootstrap.json → write 0600 → chown to the mapped guest-root → attach the read-only bind
|
|
// mount). The guest is KEPT (the golden's baked controller-bootstrap unit then deploys the baked
|
|
// controller from the mount). The per-guest token is NEVER printed (only its hash is persisted +
|
|
// the 0600 file holds the plaintext).
|
|
func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.Logger, a provisionArgs) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if a.archive == "" || a.vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires -archive <golden volid> and -vmid <N>")
|
|
return 2
|
|
}
|
|
if cfg.Backup.RestoreStorage == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires backup.restore_storage in config")
|
|
return 2
|
|
}
|
|
if a.customerID == "" || a.hubPassword == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires -customer-id and -hub-password (the customer's hub retrieval passphrase, so the controller pulls its config)")
|
|
return 2
|
|
}
|
|
if err := cfg.LocalAPI.Validate(); err != nil || !cfg.LocalAPI.Enabled() {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires local_api.enable + local_api.listen_addr in config:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
// The leaf fingerprint baked into the bootstrap must be the SAME leaf the daemon's local-API
|
|
// server serves — so use the configured (persisted) cert path. EnsureLeaf generates it once.
|
|
host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
|
|
_, fingerprint, _, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision: local-api leaf:", err)
|
|
return 1
|
|
}
|
|
tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision: token store:", err)
|
|
return 1
|
|
}
|
|
defer tokens.Close()
|
|
|
|
// --- FRONT HALF: bring up the guest (provision mode) ---
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
})
|
|
|
|
// Default the guest hostname to the customer id (DNS-safe-sanitized) when not explicitly given,
|
|
// so the CT/LXC is named meaningfully (e.g. "demo-felhom") instead of inheriting the golden's
|
|
// baked "felhom-golden". An explicit -hostname always wins.
|
|
hostname := a.hostname
|
|
if hostname == "" {
|
|
hostname = sanitizeHostname(a.customerID)
|
|
}
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=provision (vmid=%d customer=%s hostname=%s) ===\n", version, a.vmid, a.customerID, hostname)
|
|
engine.Recover(ctx)
|
|
fmt.Printf(" --- front half: bring-up (provision) %s → vmid %d ---\n", a.archive, a.vmid)
|
|
res := engine.RunBringUp(ctx, reconcile.BringUpSpec{
|
|
Mode: reconcile.ModeProvision, Archive: a.archive, VMID: a.vmid,
|
|
RestoreStorage: cfg.Backup.RestoreStorage, Hostname: hostname,
|
|
Pool: reconcile.DefaultPool, // provision into the felhom pool; required under the scoped token
|
|
Cores: a.sizing.Cores, MemoryMB: a.sizing.MemoryMB,
|
|
RootfsGrowGB: a.sizing.RootfsGrowGB, DataVolGrowGB: a.sizing.DataVolGrowGB, DataVolMount: a.sizing.DataVolMount,
|
|
SysDataGrowGB: a.sizing.SysDataGrowGB, SysDataMount: a.sizing.SysDataMount,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] front half: vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
|
|
|
|
// --- BACK HALF: mint token + populate the bootstrap config mount (host-side, no pct exec) ---
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
bh := provision.NewBackHalf(tokens, runner, "", logger)
|
|
fmt.Println(" --- back half: mint per-guest token + populate bootstrap config mount ---")
|
|
pres, err := bh.Provision(ctx, provision.Input{
|
|
VMID: a.vmid,
|
|
Customer: provision.DocCustomer{ID: a.customerID},
|
|
Hub: provision.DocHub{URL: cfg.Hub.URL, RetrievalPassword: a.hubPassword},
|
|
Endpoint: cfg.LocalAPI.ListenAddr,
|
|
Fingerprint: fingerprint,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] back-half provision (vmid %d): %v\n", a.vmid, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] back half: bootstrap mount %s → %s on vmid %d (host dir %s)\n",
|
|
pres.MountKey, pres.GuestPath, pres.VMID, pres.HostDir)
|
|
fmt.Printf(" local-api endpoint %s · leaf fp %s · token: minted (not printed)\n", cfg.LocalAPI.ListenAddr, fingerprint)
|
|
fmt.Printf("=== selftest=provision OK — guest %d provisioned + bootstrap-mounted (KEPT) ===\n", a.vmid)
|
|
fmt.Println(" next: reboot the guest → the golden's baked controller-bootstrap unit deploys the controller,")
|
|
fmt.Println(" which PULLS its controller.yaml from the hub (retrieval passphrase) and merges in this local_api.")
|
|
return 0
|
|
}
|
|
|
|
// escrowCeremonyOpts carries the CLI switches into the shared ceremony core (v0.88.0 extraction —
|
|
// the work is identical for both output modes; only the surfacing differs).
|
|
type escrowCeremonyOpts struct {
|
|
storage string
|
|
paperkey bool
|
|
offline bool
|
|
upload bool
|
|
identityBundlePath string
|
|
directivePath string
|
|
}
|
|
|
|
// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the
|
|
// --output=json wire object with RecoveryCode left EMPTY (the shells place R themselves, once).
|
|
type escrowCeremonyOutcome struct {
|
|
R string
|
|
Sum escrow.CeremonyOutput
|
|
OfflineCopy []byte // text-mode print block (opt-in b)
|
|
Paperkey string // text-mode print block (opt-in a) — SECRET-adjacent
|
|
Posture escrow.Posture
|
|
Storage string // banner data
|
|
Identity bool // banner data
|
|
}
|
|
|
|
// escrowCeremonyErr classifies a ceremony failure so both output modes keep the pre-v0.88.0 exit
|
|
// codes and stderr shapes: usage → exit 2; setup/create/upload → exit 1. kind "upload" means R
|
|
// was already minted — the text shell still surfaces it before the failure (the customer must
|
|
// receive R; the blob simply never reached the hub), exactly the pre-extraction print order.
|
|
type escrowCeremonyErr struct {
|
|
kind string // "usage" | "setup" | "create" | "upload"
|
|
err error
|
|
}
|
|
|
|
func (e *escrowCeremonyErr) Error() string { return e.err.Error() }
|
|
|
|
// escrowCeremony is the shared ceremony core (slice 7 + 10D.1 + fork-4, extracted v0.88.0 for the
|
|
// --output=json machine mode): resolve the key, assemble the identity bundle (WG key + staged
|
|
// restic password auto-attach), Create (R + self-verified blob), wipe the staged secret, upload
|
|
// when asked. It PRINTS NOTHING — the output-mode shells own every byte of stdout/stderr. R is
|
|
// returned for the caller to surface exactly once; escrow.Create never logs it and neither do we.
|
|
func escrowCeremony(ctx context.Context, cfg config.Config, logger *slog.Logger, opts escrowCeremonyOpts) (escrowCeremonyOutcome, *escrowCeremonyErr) {
|
|
var out escrowCeremonyOutcome
|
|
storage := opts.storage
|
|
if storage == "" {
|
|
storage = cfg.Escrow.PBSStorageID
|
|
}
|
|
if storage == "" {
|
|
return out, &escrowCeremonyErr{kind: "usage", err: fmt.Errorf("selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")}
|
|
}
|
|
out.Storage = storage
|
|
keyPath := cfg.Backup.PBSEncKeyPath(storage)
|
|
if _, err := os.Stat(keyPath); err != nil {
|
|
return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("PBS key for %q not found (%s): %v", storage, keyPath, err)}
|
|
}
|
|
|
|
// Slice 10D.1: optionally ALSO wrap the identity bundle under the same R, and carry the non-secret
|
|
// directive for the hub. The bundle file is a 0600 secret (tunnel/pbs tokens); the directive is
|
|
// non-secret (pbs repo/ns, expected fingerprint, tunnel id).
|
|
var identity *escrow.IdentityBundle
|
|
var directive json.RawMessage
|
|
if opts.identityBundlePath != "" {
|
|
raw, err := os.ReadFile(opts.identityBundlePath)
|
|
if err != nil {
|
|
return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("reading identity bundle %s: %v", opts.identityBundlePath, err)}
|
|
}
|
|
var b escrow.IdentityBundle
|
|
if err := json.Unmarshal(raw, &b); err != nil {
|
|
return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("identity bundle is not valid JSON {tunnel_token,pbs_token}: %v", err)}
|
|
}
|
|
identity = &b
|
|
if opts.directivePath != "" {
|
|
if d, err := os.ReadFile(opts.directivePath); err == nil && json.Valid(d) {
|
|
directive = d
|
|
}
|
|
}
|
|
}
|
|
// S3: auto-inject the offsite WG private key into the escrowed identity when the key file
|
|
// exists — a NEW escrow run should always capture the live tunnel identity. Field NAME only
|
|
// in logs, never the value. No key file + no bundle flag → pre-S3 behavior, byte-compatible.
|
|
{
|
|
wgKeyPath := wgtunnel.KeyFilePath(cfg.WGTunnel.WithDefaults().StateDir)
|
|
probe := identity
|
|
if probe == nil {
|
|
probe = &escrow.IdentityBundle{}
|
|
}
|
|
attached, err := escrow.AttachWGKey(probe, wgKeyPath)
|
|
if err != nil {
|
|
return out, &escrowCeremonyErr{kind: "setup", err: err}
|
|
}
|
|
if attached {
|
|
identity = probe
|
|
logger.Info("escrow: identity bundle: +wg_private_key")
|
|
}
|
|
}
|
|
// fork-4: auto-inject the staged offsite restic repo password (controller-pushed via the local API)
|
|
// into the escrowed identity, so DR can recover the offsite DATA key with the one recovery code R.
|
|
// Field NAME only in logs. No staged file → clean no-attach (pre-fork-4 behavior). Wiped after create.
|
|
resticStaged := false
|
|
resticPwSHA256 := "" // SLICE 3: sha256 of the sealed password (safe to upload/serve; value never logged)
|
|
{
|
|
probe := identity
|
|
if probe == nil {
|
|
probe = &escrow.IdentityBundle{}
|
|
}
|
|
attached, err := escrow.AttachResticPassword(probe, escrow.StagedResticPasswordPath())
|
|
if err != nil {
|
|
return out, &escrowCeremonyErr{kind: "setup", err: err}
|
|
}
|
|
if attached {
|
|
identity = probe
|
|
resticStaged = true
|
|
// Hash EXACTLY the value sealed into the blob — the controller matches this against
|
|
// sha256(its local repo_password) to auto-confirm the escrow (hub-verified, SLICE 3).
|
|
resticPwSHA256 = escrow.HashResticPassword(probe.ResticRepoPassword)
|
|
logger.Info("escrow: identity bundle: +restic_repo_password")
|
|
}
|
|
}
|
|
out.Identity = identity != nil
|
|
|
|
// NB: nothing about R is logged. The logger never sees R; only the shells surface it, once.
|
|
logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath, "with_identity", identity != nil)
|
|
|
|
R, res, err := escrow.Create(ctx, escrow.CreateOptions{
|
|
KeyPath: keyPath,
|
|
Posture: escrow.Posture(cfg.Escrow.Posture),
|
|
WantOfflineCopy: opts.offline,
|
|
WantPaperkey: opts.paperkey,
|
|
IdentityBundle: identity,
|
|
})
|
|
if err != nil {
|
|
return out, &escrowCeremonyErr{kind: "create", err: err}
|
|
}
|
|
// fork-4: the staged restic password is now sealed inside the R-wrapped blob — wipe the transient
|
|
// 0600 staging file so it never lingers on disk (field name only; a wipe failure is a loud warn).
|
|
if resticStaged {
|
|
if werr := escrow.WipeStagedResticPassword(); werr != nil {
|
|
logger.Warn("escrow: could not wipe the staged restic password after create", "err", werr)
|
|
}
|
|
}
|
|
|
|
out.R = R
|
|
out.Posture = res.Posture
|
|
out.OfflineCopy = res.OfflineCopy
|
|
out.Paperkey = res.Paperkey
|
|
out.Sum = escrow.CeremonyOutput{
|
|
Version: escrow.CeremonyOutputVersion,
|
|
KeyFingerprint: res.KeyFingerprint,
|
|
EntropyBits: res.EntropyBits,
|
|
BlobBytes: len(res.Blob),
|
|
IdentityBlobBytes: len(res.IdentityBlob),
|
|
ResticPwSealed: resticStaged,
|
|
}
|
|
if opts.upload {
|
|
if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil {
|
|
// R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets
|
|
// the text shell keep the pre-extraction order (R surfaced, THEN the failure).
|
|
return out, &escrowCeremonyErr{kind: "upload", err: err}
|
|
}
|
|
out.Sum.Uploaded = true
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// printEscrowTextBanner prints the text-mode header line (shared by the success and the
|
|
// post-banner failure paths so the pre-extraction output stays byte-identical — including the
|
|
// posture: the historical banner always printed the DEFAULT posture, not the result's).
|
|
func printEscrowTextBanner(out escrowCeremonyOutcome) {
|
|
fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, out.Storage, escrow.DefaultPosture, out.Identity)
|
|
}
|
|
|
|
// printEscrowTextRBlock surfaces R EXACTLY ONCE — to stdout, with the write-it-down banner —
|
|
// followed by the non-secret blob facts and the opt-in print blocks. Never logged/persisted.
|
|
func printEscrowTextRBlock(out *escrowCeremonyOutcome) {
|
|
fmt.Println()
|
|
fmt.Println(" ┌──────────────────────────────────────────────────────────────────────┐")
|
|
fmt.Println(" │ RECOVERY CODE — write it down now. It is shown ONCE and never stored. │")
|
|
fmt.Println(" │ Without it your offsite backups are unrecoverable, by anyone. │")
|
|
fmt.Println(" └──────────────────────────────────────────────────────────────────────┘")
|
|
fmt.Println(" " + out.R)
|
|
fmt.Println()
|
|
out.R = "" // drop our reference promptly
|
|
|
|
fmt.Printf(" blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n",
|
|
out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Posture, out.Sum.EntropyBits)
|
|
fmt.Println(" self-verify: the blob unwraps back to the key with R (recoverability confirmed)")
|
|
|
|
if len(out.OfflineCopy) > 0 {
|
|
fmt.Println(" --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---")
|
|
fmt.Println(base64.StdEncoding.EncodeToString(out.OfflineCopy))
|
|
}
|
|
if out.Paperkey != "" {
|
|
fmt.Println(" --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---")
|
|
fmt.Println(out.Paperkey)
|
|
}
|
|
|
|
if out.Sum.IdentityBlobBytes > 0 {
|
|
fmt.Printf(" identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", out.Sum.IdentityBlobBytes)
|
|
}
|
|
}
|
|
|
|
// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
|
|
// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
|
|
// blob. With -upload it PUTs the opaque blob to the hub. Enrollment-time, root-capable (reads the
|
|
// 0600 key). Output modes (v0.88.0):
|
|
// - text (default): the historical human output, byte-identical to pre-v0.88.0 — R to stdout
|
|
// EXACTLY ONCE inside the write-it-down banner (never to the logger/journald).
|
|
// - json: ONE machine-readable JSON object on stdout (escrow.CeremonyOutput — carries R) and
|
|
// NOTHING else there; every human/info line goes to stderr; failures exit non-zero with no
|
|
// partial JSON. This is the controller-driven ceremony's parse surface (spike §2.3: the text
|
|
// banner is positionally brittle).
|
|
func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath, outputMode string) int {
|
|
switch outputMode {
|
|
case "", "text", "json":
|
|
default:
|
|
fmt.Fprintf(os.Stderr, "selftest=escrow-create: unknown -output %q (text|json)\n", outputMode)
|
|
return 2
|
|
}
|
|
jsonMode := outputMode == "json"
|
|
if jsonMode && (offline || paperkey) {
|
|
// The opt-in print blocks are PRINT-oriented (base64 blob / paperkey text on stdout) —
|
|
// in json mode stdout carries exactly one JSON object, so refuse loudly instead of
|
|
// silently dropping what the caller asked for.
|
|
fmt.Fprintln(os.Stderr, "selftest=escrow-create: -offline/-paperkey are text-mode only (their output is print-oriented)")
|
|
return 2
|
|
}
|
|
|
|
out, cerr := escrowCeremony(ctx, cfg, logger, escrowCeremonyOpts{
|
|
storage: storage, paperkey: paperkey, offline: offline, upload: upload,
|
|
identityBundlePath: identityBundlePath, directivePath: directivePath,
|
|
})
|
|
if cerr != nil {
|
|
switch cerr.kind {
|
|
case "usage":
|
|
fmt.Fprintln(os.Stderr, cerr.err)
|
|
return 2
|
|
case "setup":
|
|
fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", cerr.err)
|
|
return 1
|
|
case "create":
|
|
if !jsonMode {
|
|
printEscrowTextBanner(out)
|
|
}
|
|
fmt.Fprintln(os.Stderr, " [FAIL] escrow create:", cerr.err)
|
|
return 1
|
|
default: // "upload" — R was minted; text mode still surfaces it (pre-extraction order)
|
|
if !jsonMode {
|
|
printEscrowTextBanner(out)
|
|
printEscrowTextRBlock(&out)
|
|
}
|
|
out.R = ""
|
|
fmt.Fprintln(os.Stderr, " [FAIL] upload escrow to hub:", cerr.err)
|
|
return 1
|
|
}
|
|
}
|
|
|
|
if jsonMode {
|
|
// Human/info lines → stderr (the job runner's diagnostics tail); the ONE JSON object with
|
|
// R → stdout. The consumer (localapi ceremony job) extracts R and zeroes its buffers.
|
|
fmt.Fprintf(os.Stderr, "=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v output=json) ===\n", version, out.Storage, out.Posture, out.Identity)
|
|
fmt.Fprintf(os.Stderr, " blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · ~%.0f bits R · self-verify OK\n",
|
|
out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Sum.EntropyBits)
|
|
if out.Sum.IdentityBlobBytes > 0 {
|
|
fmt.Fprintf(os.Stderr, " identity escrow: %d bytes · restic_pw_sealed=%v\n", out.Sum.IdentityBlobBytes, out.Sum.ResticPwSealed)
|
|
}
|
|
if out.Sum.Uploaded {
|
|
fmt.Fprintln(os.Stderr, " uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
|
|
}
|
|
wire := out.Sum
|
|
wire.RecoveryCode = out.R
|
|
if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil {
|
|
fmt.Fprintf(os.Stderr, "selftest=escrow-create: emitting JSON: %v\n", err)
|
|
return 1
|
|
}
|
|
// Best-effort scrub: drop every R reference promptly. Go's GC may retain stale copies of
|
|
// the string backing array — this shrinks the window, it cannot guarantee erasure.
|
|
wire.RecoveryCode = ""
|
|
out.R = ""
|
|
return 0
|
|
}
|
|
|
|
printEscrowTextBanner(out)
|
|
printEscrowTextRBlock(&out)
|
|
if out.Sum.Uploaded {
|
|
fmt.Println(" uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
|
|
}
|
|
fmt.Println("=== selftest=escrow-create OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestEscrowConsume exercises the slice-10C production Consume path live: recover K from an
|
|
// R-wrapped blob, gate it on the expected fingerprint, install it at -keydest. R is taken BY HAND
|
|
// from the env var FELHOM_RECOVERY_CODE (kept off the command line / ps) — never a flag, never
|
|
// logged. The other three inputs (blob/fingerprint/keydest) are flags (10D sources blob+fingerprint
|
|
// from the hub directive). This is the real Consume code, not a throwaway harness.
|
|
func runSelftestEscrowConsume(ctx context.Context, logger *slog.Logger, blobPath, expectedFP, keyDest string) int {
|
|
if blobPath == "" || expectedFP == "" || keyDest == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=escrow-consume requires -blob, -fingerprint and -keydest (R via env FELHOM_RECOVERY_CODE)")
|
|
return 2
|
|
}
|
|
R := os.Getenv("FELHOM_RECOVERY_CODE")
|
|
if R == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=escrow-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input; never a flag/arg)")
|
|
return 2
|
|
}
|
|
blob, err := os.ReadFile(blobPath)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "selftest=escrow-consume: reading blob %s: %v\n", blobPath, err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== felhom-agent %s selftest=escrow-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
|
|
logger.Info("escrow: consuming R-wrapped escrow (Unwrap → fingerprint-gate → install)",
|
|
"blob_bytes", len(blob), "key_dest", keyDest) // R is NOT logged
|
|
if err := escrow.Consume(ctx, blob, R, expectedFP, keyDest); err != nil {
|
|
R = "" // drop the reference
|
|
fmt.Fprintln(os.Stderr, " [FAIL] consume:", err) // the error never contains R or key bytes
|
|
return 1
|
|
}
|
|
R = "" // drop the reference promptly
|
|
fmt.Printf(" [OK] recovered key installed at %s (fingerprint-gated, 0600) — ready for the PBS restore\n", keyDest)
|
|
fmt.Println("=== selftest=escrow-consume OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestIdentityConsume recovers the IDENTITY bundle from its age blob with R (slice 10D.1/10D.3)
|
|
// and writes the recovered {tunnel_token, pbs_token} JSON to -keydest (0600). R is taken BY HAND from
|
|
// FELHOM_RECOVERY_CODE (off the command line); the recovered tokens are never logged. The drill then
|
|
// uses the tunnel token to re-establish the tunnel + the pbs token for steady-state.
|
|
func runSelftestIdentityConsume(ctx context.Context, cfg config.Config, logger *slog.Logger, blobPath, keyDest string, installWGKey bool) int {
|
|
if blobPath == "" || keyDest == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=identity-consume requires -blob and -keydest (R via env FELHOM_RECOVERY_CODE)")
|
|
return 2
|
|
}
|
|
R := os.Getenv("FELHOM_RECOVERY_CODE")
|
|
if R == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=identity-consume: set the recovery code in env FELHOM_RECOVERY_CODE (by-hand input)")
|
|
return 2
|
|
}
|
|
blob, err := os.ReadFile(blobPath)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "selftest=identity-consume: reading blob %s: %v\n", blobPath, err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== felhom-agent %s selftest=identity-consume (blob=%s → %s) ===\n", version, blobPath, keyDest)
|
|
logger.Info("escrow: recovering identity bundle from R-wrapped age blob", "blob_bytes", len(blob)) // R + tokens NOT logged
|
|
bundle, err := escrow.UnwrapIdentityBundle(ctx, blob, R)
|
|
if err != nil {
|
|
R = ""
|
|
fmt.Fprintln(os.Stderr, " [FAIL] identity consume:", err) // never contains R or token bytes
|
|
return 1
|
|
}
|
|
R = ""
|
|
raw, _ := json.Marshal(bundle)
|
|
if err := os.WriteFile(keyDest, raw, 0o600); err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] writing recovered bundle:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] identity recovered (tunnel_token + pbs_token) → %s (0600) — never printed\n", keyDest)
|
|
|
|
// S5 DR: install the recovered WG private key so the tunnel re-establishes with the SAME
|
|
// identity/pubkey (→ the same hub /32), no fresh keygen. Create-only (refuses to overwrite a
|
|
// present key). The VALUE is never printed — field NAME only.
|
|
if installWGKey {
|
|
stateDir := cfg.WGTunnel.WithDefaults().StateDir
|
|
switch {
|
|
case bundle.WGPrivateKey == "":
|
|
fmt.Printf(" [WARN] -install-wg-key set but the recovered bundle has NO wg_private_key (pre-S3 blob) — "+
|
|
"DR falls back to fresh keygen + re-register (keeps the /32 via hub re-key-in-place). key path: %s\n",
|
|
wgtunnel.KeyFilePath(stateDir))
|
|
default:
|
|
if err := wgtunnel.InstallRecoveredKey(stateDir, bundle.WGPrivateKey); err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] install recovered wg key:", err) // never contains the key value
|
|
return 1
|
|
}
|
|
logger.Info("escrow: installed recovered identity field", "field", "wg_private_key",
|
|
"key_path", wgtunnel.KeyFilePath(stateDir)) // NAME only, never the value
|
|
fmt.Printf(" [OK] recovered wg_private_key installed at %s (0600, create-only) — tunnel will re-establish with the same identity\n",
|
|
wgtunnel.KeyFilePath(stateDir))
|
|
}
|
|
}
|
|
fmt.Println("=== selftest=identity-consume OK ===")
|
|
return 0
|
|
}
|
|
|
|
// escrowUploadRequest is the agent→hub wire shape for the opaque escrow blob. MUST stay in lockstep
|
|
// with the hub's ingest struct (felhom-hub api.escrowUploadRequest). The hub stores the bytes and
|
|
// never decrypts them.
|
|
type escrowUploadRequest struct {
|
|
BlobB64 string `json:"blob_b64"` // base64 of the opaque R-wrapped blob (ciphertext)
|
|
KeyFingerprint string `json:"key_fingerprint"` // for operator display only
|
|
Posture string `json:"posture"` // e.g. "zero_knowledge"
|
|
// Slice 10D.1 — optional DR bundle (identity escrow + non-secret directive). Omitted in slice-7.
|
|
IdentityBlobB64 string `json:"identity_blob_b64,omitempty"`
|
|
DirectiveJSON json.RawMessage `json:"directive,omitempty"`
|
|
CreatedAt string `json:"created_at"` // RFC3339
|
|
// SLICE 3 — sha256 hex of the offsite restic repo password sealed in the identity blob (present only
|
|
// when a staged password was folded in). Non-reversible hash of a 256-bit random secret — safe to
|
|
// store/serve; lets the controller VERIFY "the escrow covers the CURRENT key" and auto-confirm.
|
|
ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"`
|
|
}
|
|
|
|
// uploadEscrowBlob PUTs the opaque blob (and, for 10D, the identity blob + non-secret directive) to
|
|
// the hub, authed with the per-host key. The hub stores ciphertext + non-secret fields; no usable
|
|
// secret leaves the agent.
|
|
func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateResult, directive json.RawMessage, resticPwSHA256 string) error {
|
|
if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
|
|
return fmt.Errorf("hub not configured (url/host_id/api_key)")
|
|
}
|
|
upReq := escrowUploadRequest{
|
|
BlobB64: base64.StdEncoding.EncodeToString(res.Blob),
|
|
KeyFingerprint: res.KeyFingerprint,
|
|
Posture: string(res.Posture),
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
ResticPwSHA256: resticPwSHA256, // "" when no staged password was folded in → omitted on the wire
|
|
}
|
|
if len(res.IdentityBlob) > 0 {
|
|
upReq.IdentityBlobB64 = base64.StdEncoding.EncodeToString(res.IdentityBlob)
|
|
upReq.DirectiveJSON = directive
|
|
}
|
|
body, _ := json.Marshal(upReq)
|
|
url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Hub.APIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
|
return fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
|
|
// path), then lists + prints the resulting PBSSnapshot records (verify-state included).
|
|
// Standalone on the host. Covers the runbook's (c) verify and (d) list.
|
|
func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version)
|
|
targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] discover pbs storages:", err)
|
|
return 1
|
|
}
|
|
if len(targets) == 0 {
|
|
fmt.Println(" no pbs storages configured on this host")
|
|
return 0
|
|
}
|
|
store := pbs.NewSnapshotStore()
|
|
loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
|
|
Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil },
|
|
Store: store,
|
|
Logger: logger,
|
|
})
|
|
// One synchronous verify+list pass over all datastores.
|
|
loop.RunOnce(ctx)
|
|
snaps := store.PBSSnapshots(ctx)
|
|
printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps)
|
|
failed := 0
|
|
for _, s := range snaps {
|
|
if s.VerifyState == pbs.VerifyFailed {
|
|
failed++
|
|
}
|
|
}
|
|
if failed > 0 {
|
|
fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets))
|
|
return 0
|
|
}
|
|
|
|
// printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
|
|
func printJSON(label string, v any) {
|
|
if b, err := json.MarshalIndent(v, " ", " "); err == nil {
|
|
fmt.Printf(" --- %s ---\n %s\n", label, string(b))
|
|
}
|
|
}
|
|
|
|
// runSelftestRead loads config, builds the API client, and runs the read-only
|
|
// queries against the live host, printing a short health report. It mutates
|
|
// nothing. Missing/invalid config is reported cleanly (no panic).
|
|
func runSelftestRead(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
|
|
return 1
|
|
}
|
|
logger.Info("selftest (read-only) starting", "config", fmt.Sprintf("%+v", cfg.Redacted().Proxmox))
|
|
|
|
client, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: client init:", err)
|
|
return 1
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest (read-only) ===\n", version)
|
|
fmt.Printf("endpoint : %s node=%s\n", cfg.Proxmox.Endpoint, cfg.Proxmox.Node)
|
|
|
|
fail := 0
|
|
report := func(label string, err error) bool {
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-14s %v\n", label, err)
|
|
fail++
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
if v, err := client.Version(ctx); report("version", err) {
|
|
fmt.Printf(" [ ok ] %-14s PVE %s (release %s)\n", "version", v.Version, v.Release)
|
|
}
|
|
if nodes, err := client.Nodes(ctx); report("nodes", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d node(s)\n", "nodes", len(nodes))
|
|
for _, n := range nodes {
|
|
marker := " "
|
|
if n.Node == cfg.Proxmox.Node {
|
|
marker = "* "
|
|
}
|
|
fmt.Printf(" %s%s status=%s fp=%s…\n", marker, n.Node, n.Status, head(n.SSLFingerprint, 17))
|
|
}
|
|
}
|
|
if s, err := client.NodeStatus(ctx); report("node status", err) {
|
|
fmt.Printf(" [ ok ] %-14s up %s, load %v, mem %s/%s, root %s/%s\n", "node status",
|
|
dur(s.Uptime), s.LoadAvg,
|
|
gib(s.Memory.Used), gib(s.Memory.Total), gib(s.RootFS.Used), gib(s.RootFS.Total))
|
|
}
|
|
if gs, err := client.ListLXC(ctx); report("list lxc", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d guest(s)\n", "list lxc", len(gs))
|
|
for _, g := range gs {
|
|
fmt.Printf(" - %d %q status=%s\n", g.VMID, g.Name, g.Status)
|
|
}
|
|
}
|
|
// A1 (v0.62.0): the stale-lock reaper's ownership registry — needs Pool.Audit (host-install v1.9.0+).
|
|
if p, err := client.Pool(ctx, reconcile.DefaultPool); report("pool read", err) {
|
|
fmt.Printf(" [ ok ] %-14s pool %q, %d member(s)\n", "pool read", p.PoolID, len(p.Members))
|
|
for _, m := range p.Members {
|
|
if m.VMID != 0 && m.Type != "storage" {
|
|
fmt.Printf(" - %d type=%s\n", m.VMID, m.Type)
|
|
}
|
|
}
|
|
}
|
|
if ss, err := client.NodeStorage(ctx); report("storage", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d store(s)\n", "storage", len(ss))
|
|
for _, s := range ss {
|
|
fmt.Printf(" - %-10s type=%-8s content=%s used=%s/%s\n",
|
|
s.Storage, s.Type, s.Content, gib(s.Used), gib(s.Total))
|
|
}
|
|
}
|
|
|
|
if fail > 0 {
|
|
fmt.Printf("=== selftest FAILED (%d check(s)) ===\n", fail)
|
|
return 1
|
|
}
|
|
fmt.Println("=== selftest OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestTask exercises WaitTask on a reversible op against -vmid: snapshot ->
|
|
// rollback -> delete-snapshot. Explicitly gated; never runs under bare --selftest.
|
|
func runSelftestTask(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
|
|
return 1
|
|
}
|
|
if vmid == 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=task requires -vmid N (a guest safe to snapshot/rollback)")
|
|
return 2
|
|
}
|
|
client, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: client init:", err)
|
|
return 1
|
|
}
|
|
|
|
// Ctrl-C aborts the wait cleanly.
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
const snap = "felhom-selftest"
|
|
steps := []struct {
|
|
name string
|
|
do func() (string, error)
|
|
}{
|
|
{"snapshot", func() (string, error) { return client.Snapshot(ctx, vmid, snap, "felhom-agent selftest") }},
|
|
{"rollback", func() (string, error) { return client.Rollback(ctx, vmid, snap) }},
|
|
{"delete-snapshot", func() (string, error) { return client.DeleteSnapshot(ctx, vmid, snap) }},
|
|
}
|
|
fmt.Printf("=== felhom-agent selftest=task (vmid %d, snapshot %q) ===\n", vmid, snap)
|
|
for _, st := range steps {
|
|
upid, err := st.do()
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", st.name, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" .... %-16s upid=%s\n", st.name, upid)
|
|
status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", st.name, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s exitstatus=%s\n", st.name, status.ExitStatus)
|
|
}
|
|
|
|
// Reversible SetConfig exercise: this is the first live use of the VM.Config.*
|
|
// privilege cluster. It round-trips the cosmetic `description` field (no runtime
|
|
// effect, fully reversible) to prove SetConfig works under the scoped token
|
|
// before slice 4's reconcile is built on top of it.
|
|
if rc := selftestSetConfig(ctx, client, vmid); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
fmt.Println("=== selftest=task OK ===")
|
|
return 0
|
|
}
|
|
|
|
// selftestSetConfig performs a reversible write+revert of the LXC `description`
|
|
// field on vmid and asserts both land. Returns 0 on success, non-zero (with a
|
|
// printed [FAIL] line) on any failure — the caller stops on a non-zero return.
|
|
func selftestSetConfig(ctx context.Context, client *proxmox.Client, vmid int) int {
|
|
// 1. Read current state; capture the original description (may be absent).
|
|
cfg, err := client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig: %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
origDesc, origPresent, err := extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description: %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
// PVE normalizes `description` by appending a trailing newline on read, so all
|
|
// comparisons here use the shared reconcile.NormDescription (strip trailing
|
|
// newlines) and restores write the normalized original — otherwise an exact-match
|
|
// check sees false drift. Same helper the slice-4 reconciler uses to normalize
|
|
// description, so the quirk has one source of truth.
|
|
origDesc = reconcile.NormDescription(origDesc)
|
|
|
|
// 2. Write the marker.
|
|
marker := "felhom-selftest " + time.Now().UTC().Format(time.RFC3339)
|
|
if rc := applySetConfig(ctx, client, vmid, "setconfig", map[string]string{"description": marker}); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
// 3. Verify the marker landed.
|
|
cfg, err = client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig (verify): %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
got, present, err := extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description (verify): %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
if !present || reconcile.NormDescription(got) != marker {
|
|
fmt.Printf(" [FAIL] %-16s write did not land: present=%v got=%q want=%q\n", "setconfig", present, got, marker)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s description verified == marker\n", "verify-write")
|
|
|
|
// 4. Restore the original value (or clear it if it was absent originally).
|
|
var revert map[string]string
|
|
if origPresent {
|
|
revert = map[string]string{"description": origDesc}
|
|
} else {
|
|
revert = map[string]string{"delete": "description"}
|
|
}
|
|
if rc := applySetConfig(ctx, client, vmid, "setconfig-revert", revert); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
// 5. Confirm the restore.
|
|
cfg, err = client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig (revert verify): %v\n", "setconfig-revert", err)
|
|
return 1
|
|
}
|
|
got, present, err = extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description (revert verify): %v\n", "setconfig-revert", err)
|
|
return 1
|
|
}
|
|
if origPresent {
|
|
if !present || reconcile.NormDescription(got) != origDesc {
|
|
fmt.Printf(" [FAIL] %-16s revert did not restore: present=%v got=%q want=%q\n", "setconfig-revert", present, reconcile.NormDescription(got), origDesc)
|
|
return 1
|
|
}
|
|
} else if present {
|
|
fmt.Printf(" [FAIL] %-16s revert did not clear: still present got=%q\n", "setconfig-revert", got)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s description restored to original\n", "verify-revert")
|
|
return 0
|
|
}
|
|
|
|
// applySetConfig runs one SetConfig and asserts success, handling PVE's dual-mode
|
|
// return: a UPID means async (WaitTask + assert exitstatus OK); an empty string
|
|
// means PVE applied it synchronously (not an error — phase1-2/mutate.go contract).
|
|
func applySetConfig(ctx context.Context, client *proxmox.Client, vmid int, step string, params map[string]string) int {
|
|
upid, err := client.SetConfig(ctx, vmid, params)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", step, err)
|
|
return 1
|
|
}
|
|
if upid == "" {
|
|
// Synchronous path: empty UPID is a clean success.
|
|
fmt.Printf(" [ ok ] %-16s synchronous exitstatus=OK\n", step)
|
|
return 0
|
|
}
|
|
status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %s %v\n", step, upid, err)
|
|
return 1
|
|
}
|
|
if status.ExitStatus != "OK" {
|
|
fmt.Printf(" [FAIL] %-16s %s exitstatus=%s\n", step, upid, status.ExitStatus)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s %s exitstatus=%s\n", step, upid, status.ExitStatus)
|
|
return 0
|
|
}
|
|
|
|
// extraString reads a string-valued key from GuestConfig.Extra (raw JSON). It
|
|
// returns ("", false, nil) when the key is absent, and decodes the JSON string
|
|
// otherwise.
|
|
func extraString(cfg proxmox.GuestConfig, key string) (string, bool, error) {
|
|
raw, ok := cfg.Extra[key]
|
|
if !ok || len(raw) == 0 {
|
|
return "", false, nil
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
return "", false, err
|
|
}
|
|
return s, true, nil
|
|
}
|
|
|
|
// --- small helpers / flag type ---
|
|
|
|
func envOr(key, def string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|
|
|
|
func head(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n]
|
|
}
|
|
|
|
func dur(seconds int64) string { return (time.Duration(seconds) * time.Second).String() }
|
|
|
|
func gib(bytes int64) string { return fmt.Sprintf("%.1fGiB", float64(bytes)/(1<<30)) }
|
|
|
|
// runSelftestLANResolver ensures dnsmasq + the host base config, then applies the split-horizon record
|
|
// for ONE guest (its live LAN IP + domain discovered from the running guest). Prints what it wrote;
|
|
// verify the actual resolution out-of-band (dig @<host-ip> <app>.<domain>). Requires a host LAN IP
|
|
// (lan_resolver.host_ip or derivable from local_api.listen_addr).
|
|
func runSelftestLANResolver(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
|
if vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=lanresolver: -vmid is required")
|
|
return 1
|
|
}
|
|
lr := cfg.LANResolver.WithDefaults(cfg.LocalAPI.ListenAddr)
|
|
if lr.HostIP == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=lanresolver: no host IP (set lan_resolver.host_ip or local_api.listen_addr)")
|
|
return 1
|
|
}
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
mgr := lanresolver.NewManager(runner, lr.HostIP, lr.Upstreams, logger)
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=lanresolver (vmid=%d host_ip=%s) ===\n", version, vmid, lr.HostIP)
|
|
if err := mgr.EnsureDnsmasq(ctx); err != nil {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] EnsureDnsmasq: %v\n", err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] dnsmasq present + base config (listen %s, upstreams %v)\n", lr.HostIP, lr.Upstreams)
|
|
cid := lanresolver.CustomerID(lr.StateDir, vmid)
|
|
if err := mgr.ReconcileGuest(ctx, vmid, cid); err != nil {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] ReconcileGuest: %v\n", err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] split-horizon applied for guest %d (customer=%q). Verify: dig @%s felhom.<domain>\n", vmid, cid, lr.HostIP)
|
|
return 0
|
|
}
|
|
|
|
// selftestFlag is a flag.Value that also satisfies IsBoolFlag, so `--selftest`
|
|
// works bare (read-only) and `--selftest=task` / `--selftest=read` set the mode.
|
|
type selftestFlag struct{ mode string }
|
|
|
|
func (f *selftestFlag) String() string { return f.mode }
|
|
func (f *selftestFlag) IsBoolFlag() bool { return true }
|
|
func (f *selftestFlag) Set(v string) error {
|
|
switch v {
|
|
case "true", "", "read":
|
|
f.mode = "read"
|
|
case "task":
|
|
f.mode = "task"
|
|
case "hub":
|
|
f.mode = "hub"
|
|
case "storage":
|
|
f.mode = "storage"
|
|
case "backup":
|
|
f.mode = "backup"
|
|
case "restore-test":
|
|
f.mode = "restore-test"
|
|
case "pbs-verify":
|
|
f.mode = "pbs-verify"
|
|
case "lanresolver":
|
|
f.mode = "lanresolver"
|
|
case "bring-up":
|
|
f.mode = "bring-up"
|
|
case "provision":
|
|
f.mode = "provision"
|
|
case "escrow-create":
|
|
f.mode = "escrow-create"
|
|
case "escrow-consume":
|
|
f.mode = "escrow-consume"
|
|
case "identity-consume":
|
|
f.mode = "identity-consume"
|
|
case "controller-swap":
|
|
f.mode = "controller-swap"
|
|
default:
|
|
return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap)", v)
|
|
}
|
|
return nil
|
|
}
|