admin
33dfd9afb3
internal/localapi: real /backup/due (cadence; due when no successful backup or newest older than backup.backup_cadence_seconds; false in-window after success; failed doesn't count) + /backup/status phases (idle|running|done|failed + job id) + POST /backup single-flight with job id. Drives the controller quiesce loop. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1477 lines
59 KiB
Go
1477 lines
59 KiB
Go
// Command felhom-agent is the host agent.
|
|
//
|
|
// With no --selftest flag it runs as the daemon: the host-report poll loop
|
|
// (slice 3) that periodically POSTs a read-only host-report to the hub (the
|
|
// heartbeat). --selftest=read|task exercise the proxmox layer; --selftest=hub does
|
|
// one collect+report against the hub and prints what it would send.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
|
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
|
)
|
|
|
|
// version is the agent version. Overridable at build time with
|
|
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
|
var version = "0.11.0"
|
|
|
|
func main() {
|
|
var (
|
|
cfgPath string
|
|
selftest selftestFlag
|
|
vmid int
|
|
watch time.Duration
|
|
archive string
|
|
mode string
|
|
hostname string
|
|
keep bool
|
|
pbsStorage string
|
|
paperkey bool
|
|
offline bool
|
|
upload bool
|
|
custID string
|
|
custDomain string
|
|
custName string
|
|
custEmail string
|
|
showVersion bool
|
|
)
|
|
flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
|
|
flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-customer-domain; keeps the guest)")
|
|
flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
|
|
flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
|
|
flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
|
|
flag.StringVar(&mode, "mode", "provision", "for --selftest=bring-up: `provision` (golden, fresh identity) | `dr` (customer backup, preserve continuity)")
|
|
flag.StringVar(&hostname, "hostname", "", "for --selftest=bring-up provision: the hostname to set on the new guest")
|
|
flag.BoolVar(&keep, "keep", false, "for --selftest=bring-up: KEEP the guest instead of tearing it down at the end")
|
|
flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
|
|
flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
|
|
flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
|
|
flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
|
|
flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id to seed into the guest's bootstrap")
|
|
flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: the customer domain to seed")
|
|
flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: the customer display name to seed (optional)")
|
|
flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: the customer email to seed (optional)")
|
|
flag.BoolVar(&showVersion, "version", false, "print version and exit")
|
|
flag.Parse()
|
|
|
|
if showVersion {
|
|
fmt.Println("felhom-agent", version)
|
|
return
|
|
}
|
|
|
|
cfg, err := config.Load(cfgPath)
|
|
if err != nil {
|
|
// A missing default config file is fine if env provides the values; only a
|
|
// present-but-unreadable/invalid file is fatal here.
|
|
if !(os.IsNotExist(errors.Unwrap(err)) && cfgPath == flag.Lookup("config").DefValue) {
|
|
fmt.Fprintln(os.Stderr, "config error:", err)
|
|
os.Exit(2)
|
|
}
|
|
cfg = config.Default()
|
|
}
|
|
logger := applog.New(cfg.LogLevel)
|
|
|
|
switch selftest.mode {
|
|
case "":
|
|
os.Exit(runDaemon(cfg, logger))
|
|
case "read":
|
|
os.Exit(runSelftestRead(context.Background(), cfg, logger))
|
|
case "task":
|
|
os.Exit(runSelftestTask(context.Background(), cfg, logger, vmid))
|
|
case "hub":
|
|
os.Exit(runSelftestHub(context.Background(), cfg, logger))
|
|
case "storage":
|
|
os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
|
|
case "backup":
|
|
os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
|
|
case "restore-test":
|
|
os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
|
|
case "pbs-verify":
|
|
os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
|
|
case "bring-up":
|
|
os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep))
|
|
case "provision":
|
|
os.Exit(runSelftestProvision(context.Background(), cfg, logger, provisionArgs{
|
|
archive: archive, vmid: vmid, hostname: hostname,
|
|
customer: provision.DocCustomer{ID: custID, Domain: custDomain, Name: custName, Email: custEmail},
|
|
}))
|
|
case "escrow-create":
|
|
os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload))
|
|
}
|
|
}
|
|
|
|
// newProxmoxClient builds the read-path proxmox client from config.
|
|
func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) {
|
|
return proxmox.NewClient(proxmox.Config{
|
|
Endpoint: cfg.Proxmox.Endpoint,
|
|
Node: cfg.Proxmox.Node,
|
|
Token: cfg.Proxmox.Token,
|
|
TLS: proxmox.TLSConfig{
|
|
CAFile: cfg.Proxmox.TLS.CAFile,
|
|
Fingerprint: cfg.Proxmox.TLS.Fingerprint,
|
|
InsecureSkipVerify: cfg.Proxmox.TLS.InsecureSkipVerify,
|
|
},
|
|
})
|
|
}
|
|
|
|
// newHostOps builds the privileged storage surface (slice 5 Phase B) from config. It shells
|
|
// out via the same fenced Runner the proxmox layer uses (sudo -n, arg vectors, no shell);
|
|
// every argument is validated in internal/storage before any command is built. A
|
|
// missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a
|
|
// crash.
|
|
func newHostOps(cfg config.Config, logger *slog.Logger) storage.HostOps {
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
return storage.NewSudoHostOps(storage.SudoHostOpsConfig{
|
|
Runner: runner,
|
|
Bins: storage.Binaries{
|
|
Systemctl: cfg.Privileged.Systemctl,
|
|
Install: cfg.Privileged.Install,
|
|
Smartctl: cfg.Privileged.Smartctl,
|
|
Lvs: cfg.Privileged.Lvs,
|
|
},
|
|
UnitDir: cfg.Privileged.UnitDir,
|
|
StageDir: cfg.Privileged.StageDir,
|
|
Logger: logger,
|
|
})
|
|
}
|
|
|
|
// gateRemounter is the watchdog's benign re-mount response: it routes the re-mount through
|
|
// the reversibility gate (classified benign) and, if allowed, calls HostOps.EnsureMount. It
|
|
// lives here (not in internal/storage) so storage stays decoupled from reconcile — main is
|
|
// the one place that holds both.
|
|
type gateRemounter struct {
|
|
gate *reconcile.Gate
|
|
ops storage.HostOps
|
|
hostID string
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// Remount authorizes (benign) and performs a by-UUID re-mount of a returned target.
|
|
func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
|
|
dec := r.gate.Authorize(reconcile.IntentForStorageMount(r.hostID, t.Name), nil)
|
|
if !dec.Allowed {
|
|
r.logger.Warn("storage: re-mount refused by gate (unexpected for a benign mount)",
|
|
"target", t.Name, "reason", dec.Reason)
|
|
return
|
|
}
|
|
uuid := t.UUID
|
|
if uuid == "" {
|
|
uuid = strings.TrimPrefix(t.DurableID, "uuid:") // durable_id carries it for usb/local-dir
|
|
}
|
|
spec := storage.MountSpec{Name: t.Name, UUID: uuid, Where: t.MountPath}
|
|
if err := r.ops.EnsureMount(ctx, spec); err != nil {
|
|
r.logger.Error("storage: re-mount failed", "target", t.Name, "where", t.MountPath, "err", err)
|
|
return
|
|
}
|
|
r.logger.Info("storage: re-mounted returned target", "target", t.Name, "where", t.MountPath)
|
|
}
|
|
|
|
// runDaemon is the default mode: collect a host-report and POST it to the hub on a
|
|
// loop. Requires both proxmox (to collect) and hub config.
|
|
func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: proxmox not configured:", err)
|
|
return 2
|
|
}
|
|
if err := cfg.Hub.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: hub not configured:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: proxmox client:", err)
|
|
return 1
|
|
}
|
|
client, err := hub.NewClient(cfg.Hub, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: hub client:", err)
|
|
return 1
|
|
}
|
|
hcfg := cfg.Hub.WithDefaults()
|
|
// Storage observer (slice 5): builds the report's storage_targets from Proxmox +
|
|
// non-privileged host reads, enriched with the privileged SMART/lvs reads via HostOps
|
|
// (Phase B). Wired into the collector via the StorageObserver seam (so hub does not
|
|
// import storage).
|
|
hostReader := storage.NewProcHostReader()
|
|
hostOps := newHostOps(cfg, logger)
|
|
observer := storage.NewObserver(px, hostReader, hostOps, logger)
|
|
// Backup + restore-test state store (slice 6): holds the latest backup-per-target +
|
|
// latest restore-test result; the collector reads it via the BackupReporter /
|
|
// RestoreTestReporter seams; the cadence scheduler writes it.
|
|
backupStore := backup.NewStore()
|
|
// PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes it; the
|
|
// collector reads it via the PBSReporter seam.
|
|
pbsStore := pbs.NewSnapshotStore()
|
|
collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsStore, cfg.Hub.HostID, version, logger)
|
|
loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
|
|
interval := time.Duration(hcfg.PollSeconds) * time.Second
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
logger.Info("felhom-agent daemon starting",
|
|
"version", version, "host_id", cfg.Hub.HostID, "hub_url", cfg.Hub.URL,
|
|
"interval_s", hcfg.PollSeconds) // hub key intentionally not logged
|
|
|
|
// Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue
|
|
// (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving
|
|
// until slice 10), so reconcile is a live no-op: it reads state and computes an
|
|
// empty action set each tick, mutating nothing. The daemon must run cleanly with
|
|
// no desired state and no signers configured — so a journal-open failure is logged
|
|
// and reconcile proceeds journal-less (it has nothing destructive to journal yet).
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err != nil {
|
|
logger.Warn("daemon: cannot ensure journal dir; reconcile runs without a journal", "path", jp, "err", err)
|
|
} else if j, err := reconcile.OpenJournal(jp); err != nil {
|
|
logger.Warn("daemon: cannot open op journal; reconcile runs without a journal", "path", jp, "err", err)
|
|
} else {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
// The reversibility gate (slice 4 Phase B) sits in front of every mutation. With
|
|
// no signers pinned (the common slice-4 state) the verifier is nil: benign actions
|
|
// pass, destructive intents are refused pending_signature — and reconcile only
|
|
// produces benign actions, so nothing is gated away. A misconfigured signer key is
|
|
// a security misconfig and is fatal.
|
|
verifier, nonceStore, err := buildVerifier(cfg, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "daemon: authz verifier:", err)
|
|
return 2
|
|
}
|
|
if nonceStore != nil {
|
|
defer nonceStore.Close()
|
|
}
|
|
gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
|
|
// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
|
|
// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
|
|
// known mount-backed target's device returning unmounted, dispatches a benign re-mount
|
|
// (routed through the gate as benign, then HostOps). With no removable/network storage
|
|
// it finds nothing to flag. The re-mount dispatch is off the poll path (a goroutine).
|
|
storageTrigger := make(chan struct{}, 1)
|
|
loop.SetTrigger(storageTrigger)
|
|
remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
|
|
watchdog := storage.NewWatchdog(storage.WatchdogOptions{
|
|
Targets: storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
|
|
Liveness: storage.NewHostLiveness(hostReader, 0),
|
|
Remounter: remounter,
|
|
Trigger: func() {
|
|
select {
|
|
case storageTrigger <- struct{}{}:
|
|
default:
|
|
}
|
|
},
|
|
Interval: cfg.Storage.WatchdogInterval(),
|
|
Debounce: cfg.Storage.WatchdogDebounce(),
|
|
Logger: logger,
|
|
})
|
|
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px,
|
|
Queue: queue,
|
|
Journal: journal,
|
|
Provider: reconcile.EmptyProvider{}, // slice 4: no live desired-state source
|
|
Gate: gate,
|
|
HostID: cfg.Hub.HostID,
|
|
Logger: logger,
|
|
})
|
|
|
|
// Crash recovery (doc 03 §10): resolve any op that was in flight when the agent
|
|
// last died BEFORE issuing new mutations. For slice 6 this is load-bearing — a
|
|
// restore-test scratch guest leaked by a mid-test crash is torn down here.
|
|
engine.Recover(ctx)
|
|
|
|
// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
|
|
// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
|
|
// OR the scratch band / restore storage is misconfigured — the daemon still runs.
|
|
scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)
|
|
|
|
// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
|
|
// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
|
|
// verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs
|
|
// storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing.
|
|
pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
|
|
Targets: pbsTargetsFromPVE(cfg, px, logger),
|
|
Store: pbsStore,
|
|
Cadence: cfg.Backup.PBSVerifyCadence(),
|
|
Logger: logger,
|
|
})
|
|
|
|
// Local API server (slice 8A, doc 03 §6): the per-guest authorization gate the in-guest
|
|
// controller calls over the bridge. Optional — runs only when local_api.enable + listen_addr
|
|
// are configured; a token-store or cert failure disables it WITHOUT killing the daemon (the
|
|
// host still reports/reconciles). The leaf is generated+persisted once so its pin is stable.
|
|
localServers := 0
|
|
var localTokens *localapi.TokenStore
|
|
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, logger, &localTokens)
|
|
if localTokens != nil {
|
|
defer localTokens.Close()
|
|
}
|
|
|
|
// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
|
|
// verify loop, and (optionally) the local-API server concurrently; any one returning ends
|
|
// the daemon (ctx cancel tears down the rest).
|
|
errc := make(chan error, 6)
|
|
go func() { errc <- engine.Run(ctx, interval) }()
|
|
go func() { errc <- loop.Run(ctx) }()
|
|
go func() { errc <- watchdog.Run(ctx) }()
|
|
go func() { errc <- scheduler.Run(ctx) }()
|
|
go func() { errc <- pbsLoop.Run(ctx) }()
|
|
if localSrv != nil {
|
|
localServers = 1
|
|
go func() { errc <- localSrv.Run(ctx) }()
|
|
}
|
|
|
|
err = <-errc
|
|
stop() // tear down the siblings on the first exit
|
|
for i := 0; i < 4+localServers; i++ { // wait for the other goroutines
|
|
<-errc
|
|
}
|
|
if err != nil && err != context.Canceled {
|
|
logger.Error("daemon: exited with error", "err", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs
|
|
// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each
|
|
// (token id from the storage `username`, secret read from <PBSSecretDir>/<id>.pw). A storage
|
|
// whose secret/client can't be built is skipped with a warning (the loop still verifies the
|
|
// rest). The secret is read at runtime, never logged.
|
|
func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets {
|
|
return func(ctx context.Context) ([]pbs.Target, error) {
|
|
stores, err := px.ListStorage(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var targets []pbs.Target
|
|
for _, s := range stores {
|
|
if s.Type != "pbs" {
|
|
continue
|
|
}
|
|
secret, err := readTrimmed(cfg.Backup.PBSSecretPath(s.Storage))
|
|
if err != nil {
|
|
logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
|
|
continue
|
|
}
|
|
c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret})
|
|
if err != nil {
|
|
logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
|
|
continue
|
|
}
|
|
targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c})
|
|
}
|
|
return targets, nil
|
|
}
|
|
}
|
|
|
|
// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
|
|
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
|
|
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
|
|
stores, err := px.ListStorage(ctx)
|
|
if err != nil {
|
|
return "local"
|
|
}
|
|
for _, s := range stores {
|
|
if s.Storage == storageID && s.Type == "pbs" {
|
|
return "pbs"
|
|
}
|
|
}
|
|
return "local"
|
|
}
|
|
|
|
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
|
|
func readTrimmed(path string) (string, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
s := strings.TrimSpace(string(b))
|
|
if s == "" {
|
|
return "", fmt.Errorf("empty file %s", path)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
|
|
// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
|
|
// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
|
|
// machinery still works on-demand via --selftest=restore-test.
|
|
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, logger *slog.Logger) *backup.Scheduler {
|
|
cadence := cfg.Backup.RestoreTestCadence()
|
|
if cadence > 0 {
|
|
if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
|
|
logger.Warn("daemon: restore-test cadence disabled (config invalid)", "err", err)
|
|
cadence = 0
|
|
}
|
|
}
|
|
min, max := cfg.Backup.ScratchBand()
|
|
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom restore-test", logger)
|
|
return backup.NewScheduler(backup.SchedulerOptions{
|
|
Runner: engine,
|
|
Pick: runner.PickRestoreCandidate,
|
|
Store: store,
|
|
Spec: reconcile.RestoreTestSpec{
|
|
RestoreStorage: cfg.Backup.RestoreStorage,
|
|
ScratchMin: min,
|
|
ScratchMax: max,
|
|
SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget),
|
|
},
|
|
Cadence: cadence,
|
|
Logger: logger,
|
|
})
|
|
}
|
|
|
|
// buildLocalAPIServer constructs the per-guest local-API server (slice 8A, doc 03 §6) when
|
|
// configured. It opens the durable hashed token store and ensures the persisted self-signed
|
|
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
|
|
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
|
|
// fixed. The opened token store is returned via outTokens so the caller can Close it.
|
|
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
|
|
if !cfg.LocalAPI.Enabled() {
|
|
return nil
|
|
}
|
|
if err := cfg.LocalAPI.Validate(); err != nil {
|
|
logger.Warn("daemon: local-api disabled (config invalid)", "err", err)
|
|
return nil
|
|
}
|
|
tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (token store)", "err", err)
|
|
return nil
|
|
}
|
|
*outTokens = tokens
|
|
host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
|
|
cert, fp, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (leaf cert)", "err", err)
|
|
return nil
|
|
}
|
|
logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
|
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger)
|
|
srv, err := localapi.NewServer(localapi.Options{
|
|
ListenAddr: cfg.LocalAPI.ListenAddr,
|
|
Cert: cert,
|
|
Guests: px,
|
|
Backups: runner,
|
|
Store: store,
|
|
Storage: observer,
|
|
Tokens: tokens,
|
|
BackupCadence: cfg.Backup.BackupCadence(),
|
|
Logger: logger,
|
|
})
|
|
if err != nil {
|
|
logger.Warn("daemon: local-api disabled (server build)", "err", err)
|
|
return nil
|
|
}
|
|
return srv
|
|
}
|
|
|
|
// reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
|
|
// configured nonce store (both are durable agent state), falling back to the standard
|
|
// host state dir when the nonce store is unset.
|
|
func reconcileJournalPath(cfg config.Config) string {
|
|
if p := cfg.Authz.NonceStorePath; p != "" {
|
|
return filepath.Join(filepath.Dir(p), "journal.log")
|
|
}
|
|
return "/var/lib/felhom-agent/journal.log"
|
|
}
|
|
|
|
// buildVerifier constructs the operator-signed-op verifier from config. With no signers
|
|
// pinned it returns (nil, nil, nil) — the gate then refuses any destructive intent as
|
|
// pending_signature (and reconcile serves none). With signers it requires a durable
|
|
// nonce-store path (anti-replay must survive restarts) and parses every pinned key —
|
|
// a bad key or missing path is a fatal security misconfig.
|
|
func buildVerifier(cfg config.Config, logger *slog.Logger) (reconcile.OpVerifier, *authz.FileNonceStore, error) {
|
|
if len(cfg.Authz.Signers) == 0 {
|
|
logger.Info("daemon: no operator signers pinned; destructive ops will be refused pending_signature")
|
|
return nil, nil, nil
|
|
}
|
|
if cfg.Authz.NonceStorePath == "" {
|
|
return nil, nil, fmt.Errorf("authz.nonce_store_path is required when signers are configured")
|
|
}
|
|
signers := make([]authz.AllowedSigner, 0, len(cfg.Authz.Signers))
|
|
for _, s := range cfg.Authz.Signers {
|
|
as, err := authz.NewAllowedSigner(s.KeyID, authz.KeyRole(s.Role), s.PublicKey)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("pinned signer %q: %w", s.KeyID, err)
|
|
}
|
|
signers = append(signers, as)
|
|
}
|
|
store, err := authz.OpenFileNonceStore(cfg.Authz.NonceStorePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("nonce store: %w", err)
|
|
}
|
|
logger.Info("daemon: operator signers pinned", "count", len(signers))
|
|
return authz.New(signers, store, cfg.Hub.HostID), store, nil
|
|
}
|
|
|
|
// runSelftestHub validates hub config, does ONE collect + report, and prints the
|
|
// report it would send plus the envelope it got back.
|
|
func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if err := cfg.Hub.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: hub not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
client, err := hub.NewClient(cfg.Hub, logger)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: hub client:", err)
|
|
return 1
|
|
}
|
|
observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
|
|
collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, nil, cfg.Hub.HostID, version, logger)
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=hub (host_id=%s url=%s) ===\n", version, cfg.Hub.HostID, cfg.Hub.URL)
|
|
report, err := collector.Collect(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] collect:", err)
|
|
return 1
|
|
}
|
|
if b, e := json.MarshalIndent(report, " ", " "); e == nil {
|
|
fmt.Println(" --- report it would send ---")
|
|
fmt.Println(" " + string(b))
|
|
}
|
|
env, err := client.Report(ctx, report)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] report:", err)
|
|
return 1
|
|
}
|
|
if b, e := json.MarshalIndent(env, " ", " "); e == nil {
|
|
fmt.Println(" --- control envelope received ---")
|
|
fmt.Println(" " + string(b))
|
|
}
|
|
fmt.Println("=== selftest=hub OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestStorage is the live storage harness (slice 5 Phase B, for the USB runbook).
|
|
// It needs Proxmox config only (NO hub) so it runs standalone on the Proxmox host:
|
|
// - observe pass: print the full StorageTarget table incl. the privileged SMART summary
|
|
// and thin-pool data+metadata fill.
|
|
// - -watch D: run the watchdog verbose for D with the re-mount response LIVE, so the
|
|
// operator can physically cycle a drive and watch detect → report → re-mount.
|
|
func runSelftestStorage(ctx context.Context, cfg config.Config, logger *slog.Logger, watch time.Duration) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
hostReader := storage.NewProcHostReader()
|
|
hostOps := newHostOps(cfg, logger)
|
|
observer := storage.NewObserver(px, hostReader, hostOps, logger)
|
|
|
|
octx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
|
fmt.Printf("=== felhom-agent %s selftest=storage ===\n", version)
|
|
targets, err := observer.Observe(octx)
|
|
cancel()
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] observe:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" observed %d storage target(s):\n", len(targets))
|
|
for _, t := range targets {
|
|
fmt.Printf(" - %-12s type=%-9s state=%-12s reach=%-5v class=%-4s durable=%s\n",
|
|
t.Name, t.Type, t.State, t.Reachable, t.ClassHint, t.DurableID)
|
|
fmt.Printf(" usage %s/%s (%.0f%%) mount=%q dev=%q\n",
|
|
gib(t.UsedBytes), gib(t.TotalBytes), t.UsedFraction*100, t.MountPath, t.BackingDevice)
|
|
fmt.Printf(" smart: health=%s%s\n", t.Smart.Health, smartCounters(t.Smart))
|
|
if t.ThinPool != nil {
|
|
meta := "n/a"
|
|
if t.ThinPool.MetadataUsedFraction != nil {
|
|
meta = fmt.Sprintf("%.1f%%", *t.ThinPool.MetadataUsedFraction*100)
|
|
}
|
|
fmt.Printf(" thin-pool: data=%.1f%% metadata=%s\n", t.ThinPool.DataUsedFraction*100, meta)
|
|
}
|
|
}
|
|
|
|
if watch <= 0 {
|
|
fmt.Println("=== selftest=storage OK (observe pass; pass -watch D for the live watchdog) ===")
|
|
return 0
|
|
}
|
|
|
|
// Live watchdog window: re-mount response active. Cycle a drive and watch the logs.
|
|
fmt.Printf(" --- watching for %s (cycle a drive now; detect → report → re-mount) ---\n", watch)
|
|
wctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
wctx, cancel2 := context.WithTimeout(wctx, watch)
|
|
defer cancel2()
|
|
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
|
|
wd := storage.NewWatchdog(storage.WatchdogOptions{
|
|
Targets: storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
|
|
Liveness: storage.NewHostLiveness(hostReader, 0),
|
|
Remounter: remounter,
|
|
Trigger: func() { logger.Info("storage: (selftest) would send out-of-band host-report now") },
|
|
Interval: cfg.Storage.WatchdogInterval(),
|
|
Debounce: cfg.Storage.WatchdogDebounce(),
|
|
Logger: logger,
|
|
})
|
|
_ = wd.Run(wctx)
|
|
fmt.Println("=== selftest=storage watch window ended ===")
|
|
return 0
|
|
}
|
|
|
|
// smartCounters renders the non-nil SMART counters compactly for the selftest table.
|
|
func smartCounters(s hub.SmartSummary) string {
|
|
var parts []string
|
|
add := func(name string, v *int) {
|
|
if v != nil {
|
|
parts = append(parts, fmt.Sprintf("%s=%d", name, *v))
|
|
}
|
|
}
|
|
add("temp", s.TemperatureC)
|
|
add("poh", s.PowerOnHours)
|
|
add("realloc", s.ReallocatedSectors)
|
|
add("pending", s.PendingSectors)
|
|
add("offline_unc", s.OfflineUncorrectable)
|
|
add("crit_warn", s.CriticalWarning)
|
|
add("media_err", s.MediaErrors)
|
|
add("pct_used", s.PercentageUsed)
|
|
if len(parts) == 0 {
|
|
return ""
|
|
}
|
|
return " " + strings.Join(parts, " ")
|
|
}
|
|
|
|
// runSelftestBackup runs one vzdump of -vmid to the configured local backup target and
|
|
// prints the resulting Backup record. Standalone (no hub). Live + benign.
|
|
func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if vmid == 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
|
|
return 2
|
|
}
|
|
if cfg.Backup.LocalBackupTarget == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=backup requires backup.local_backup_target in config (a content=backup storage)")
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, cfg.Backup.LocalBackupTarget)
|
|
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom selftest", logger)
|
|
rec, err := runner.Backup(ctx, vmid)
|
|
printJSON("backup record", rec)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] backup:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=backup OK (crash-consistent=%v, archive=%s) ===\n", rec.CrashConsistent, rec.Archive)
|
|
return 0
|
|
}
|
|
|
|
// runSelftestRestoreTest runs one self-restore-test (restore → net-link-down → boot → verify
|
|
// running → teardown) of -archive (or the newest backup on the local target) into a scratch
|
|
// guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
|
|
// crashed test is reaped before this run.
|
|
func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=restore-test:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
// Build a journaled engine so a leaked scratch from a prior crash is recoverable.
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
})
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=restore-test ===\n", version)
|
|
fmt.Println(" --- recover: reaping any leaked scratch from a prior crashed test ---")
|
|
rec := engine.Recover(ctx)
|
|
fmt.Printf(" recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)
|
|
|
|
if archive == "" {
|
|
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "", logger)
|
|
archive, err = runner.PickRestoreCandidate(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] pick backup:", err)
|
|
return 1
|
|
}
|
|
if archive == "" {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", cfg.Backup.LocalBackupTarget, "(run --selftest=backup first)")
|
|
return 1
|
|
}
|
|
}
|
|
min, max := cfg.Backup.ScratchBand()
|
|
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
|
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
|
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
|
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget),
|
|
})
|
|
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
|
if res.Skipped {
|
|
fmt.Println("=== selftest=restore-test SKIPPED (no free scratch VMID in band) ===")
|
|
return 0
|
|
}
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] restore-test (scratch %d): %v\n", res.ScratchVMID, res.Err)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=restore-test OK (scratch %d restored+booted+verified+torn-down in %s) ===\n", res.ScratchVMID, res.Duration.Round(time.Second))
|
|
return 0
|
|
}
|
|
|
|
// runSelftestBringUp runs the REAL journaled bring-up job (slice 7) on-demand: restore →
|
|
// reset identity → size → attach mounts → start link-up, then tears the guest down by default
|
|
// (a selftest must not leave a guest running) unless -keep. -mode picks provision (golden, fresh
|
|
// identity) or dr (customer backup, preserve continuity). It first Recovers, so a leaked guest
|
|
// from a prior crashed bring-up is reaped before this run.
|
|
func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Logger, mode, archive string, vmid int, hostname string, keep bool) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
var bmode reconcile.BringUpMode
|
|
switch mode {
|
|
case "provision":
|
|
bmode = reconcile.ModeProvision
|
|
case "dr":
|
|
bmode = reconcile.ModeDRGuestLoss
|
|
default:
|
|
fmt.Fprintf(os.Stderr, "selftest=bring-up: -mode must be provision|dr (got %q)\n", mode)
|
|
return 2
|
|
}
|
|
if archive == "" || vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=bring-up requires -archive <volid> and -vmid <N>")
|
|
return 2
|
|
}
|
|
if cfg.Backup.RestoreStorage == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=bring-up requires backup.restore_storage in config")
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
})
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
|
|
fmt.Println(" --- recover: reaping any half-built guest from a prior crashed bring-up ---")
|
|
rec := engine.Recover(ctx)
|
|
fmt.Printf(" recover: examined=%d bring_up_rolled_back=%d bring_up_clean=%d scratch_destroyed=%d\n",
|
|
rec.Examined, rec.BringUpRolledBack, rec.BringUpClean, rec.ScratchDestroyed)
|
|
|
|
spec := reconcile.BringUpSpec{
|
|
Mode: bmode, Archive: archive, VMID: vmid, RestoreStorage: cfg.Backup.RestoreStorage,
|
|
Hostname: hostname, KeepMAC: bmode == reconcile.ModeDRGuestLoss,
|
|
}
|
|
fmt.Printf(" bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
|
|
res := engine.RunBringUp(ctx, spec)
|
|
printJSON("bring-up record", res)
|
|
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] bring-up (vmid %d): %v\n", vmid, res.Err)
|
|
// On failure the job already compensating-rolled-back; nothing to tear down.
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
|
|
if len(res.StartWarnings) > 0 {
|
|
fmt.Printf(" start warnings (recognized=%v): %v\n", res.WarningsRecognized, res.StartWarnings)
|
|
}
|
|
|
|
if keep {
|
|
fmt.Printf("=== selftest=bring-up OK — guest %d KEPT (-keep) ===\n", vmid)
|
|
return 0
|
|
}
|
|
// A selftest must not leave a guest running. Tear it down (out-of-band, like the spike).
|
|
fmt.Printf(" --- teardown: destroying selftest guest %d ---\n", vmid)
|
|
upid, err := px.DestroyLXC(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " [WARN] teardown destroy: %v (destroy vmid %d manually)\n", err, vmid)
|
|
return 1
|
|
}
|
|
if upid != "" {
|
|
if _, err := px.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
|
fmt.Fprintf(os.Stderr, " [WARN] teardown destroy task: %v (check vmid %d)\n", err, vmid)
|
|
return 1
|
|
}
|
|
}
|
|
fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
|
|
return 0
|
|
}
|
|
|
|
// provisionArgs bundles the --selftest=provision inputs.
|
|
type provisionArgs struct {
|
|
archive string
|
|
vmid int
|
|
hostname string
|
|
customer provision.DocCustomer
|
|
}
|
|
|
|
// runSelftestProvision runs the FULL slice-8A provisioning chain on-demand: the slice-7 bring-up
|
|
// FRONT half (provision mode, golden) + the slice-8A BACK half (mint per-guest token → render the
|
|
// stable bootstrap.json → write 0600 → chown to the mapped guest-root → attach the read-only bind
|
|
// mount). The guest is KEPT (the golden's baked controller-bootstrap unit then deploys the baked
|
|
// controller from the mount). The per-guest token is NEVER printed (only its hash is persisted +
|
|
// the 0600 file holds the plaintext).
|
|
func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.Logger, a provisionArgs) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
if a.archive == "" || a.vmid <= 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires -archive <golden volid> and -vmid <N>")
|
|
return 2
|
|
}
|
|
if cfg.Backup.RestoreStorage == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires backup.restore_storage in config")
|
|
return 2
|
|
}
|
|
if a.customer.ID == "" || a.customer.Domain == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires -customer-id and -customer-domain (so the controller skips setup)")
|
|
return 2
|
|
}
|
|
if err := cfg.LocalAPI.Validate(); err != nil || !cfg.LocalAPI.Enabled() {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision requires local_api.enable + local_api.listen_addr in config:", err)
|
|
return 2
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
// The leaf fingerprint baked into the bootstrap must be the SAME leaf the daemon's local-API
|
|
// server serves — so use the configured (persisted) cert path. EnsureLeaf generates it once.
|
|
host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
|
|
_, fingerprint, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision: local-api leaf:", err)
|
|
return 1
|
|
}
|
|
tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest=provision: token store:", err)
|
|
return 1
|
|
}
|
|
defer tokens.Close()
|
|
|
|
// --- FRONT HALF: bring up the guest (provision mode) ---
|
|
queue := reconcile.NewQueue()
|
|
defer queue.Close()
|
|
var journal *reconcile.Journal
|
|
if jp := reconcileJournalPath(cfg); jp != "" {
|
|
if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
|
|
if j, err := reconcile.OpenJournal(jp); err == nil {
|
|
journal = j
|
|
defer journal.Close()
|
|
}
|
|
}
|
|
}
|
|
gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
|
engine := reconcile.NewEngine(reconcile.EngineOptions{
|
|
API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
|
|
})
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=provision (vmid=%d customer=%s) ===\n", version, a.vmid, a.customer.ID)
|
|
engine.Recover(ctx)
|
|
fmt.Printf(" --- front half: bring-up (provision) %s → vmid %d ---\n", a.archive, a.vmid)
|
|
res := engine.RunBringUp(ctx, reconcile.BringUpSpec{
|
|
Mode: reconcile.ModeProvision, Archive: a.archive, VMID: a.vmid,
|
|
RestoreStorage: cfg.Backup.RestoreStorage, Hostname: a.hostname,
|
|
})
|
|
if res.Err != nil || !res.Pass {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] front half: vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
|
|
|
|
// --- BACK HALF: mint token + populate the bootstrap config mount (host-side, no pct exec) ---
|
|
mode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
|
if mode == "" {
|
|
mode = proxmox.RunnerSudo
|
|
}
|
|
runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
|
|
bh := provision.NewBackHalf(tokens, runner, "", logger)
|
|
fmt.Println(" --- back half: mint per-guest token + populate bootstrap config mount ---")
|
|
pres, err := bh.Provision(ctx, provision.Input{
|
|
VMID: a.vmid,
|
|
Customer: a.customer,
|
|
Hub: provision.DocHub{URL: cfg.Hub.URL, APIKey: cfg.Hub.APIKey, HostID: cfg.Hub.HostID},
|
|
Endpoint: cfg.LocalAPI.ListenAddr,
|
|
Fingerprint: fingerprint,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " [FAIL] back-half provision (vmid %d): %v\n", a.vmid, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [OK] back half: bootstrap mount %s → %s on vmid %d (host dir %s)\n",
|
|
pres.MountKey, pres.GuestPath, pres.VMID, pres.HostDir)
|
|
fmt.Printf(" local-api endpoint %s · leaf fp %s · token: minted (not printed)\n", cfg.LocalAPI.ListenAddr, fingerprint)
|
|
fmt.Printf("=== selftest=provision OK — guest %d provisioned + bootstrap-mounted (KEPT) ===\n", a.vmid)
|
|
fmt.Println(" next: the golden's baked controller-bootstrap unit deploys the controller from the mount on boot.")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
|
|
// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
|
|
// blob. R is surfaced to stdout EXACTLY ONCE (never to the logger/journald). With -upload it PUTs
|
|
// the opaque blob to the hub. Enrollment-time, root-capable (reads the 0600 key).
|
|
func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool) int {
|
|
if storage == "" {
|
|
storage = cfg.Escrow.PBSStorageID
|
|
}
|
|
if storage == "" {
|
|
fmt.Fprintln(os.Stderr, "selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")
|
|
return 2
|
|
}
|
|
keyPath := cfg.Backup.PBSEncKeyPath(storage)
|
|
if _, err := os.Stat(keyPath); err != nil {
|
|
fmt.Fprintf(os.Stderr, "selftest=escrow-create: PBS key for %q not found (%s): %v\n", storage, keyPath, err)
|
|
return 1
|
|
}
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s) ===\n", version, storage, escrow.DefaultPosture)
|
|
// NB: nothing about R is logged. The logger never sees R; only stdout does, once.
|
|
logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath)
|
|
|
|
R, res, err := escrow.Create(ctx, escrow.CreateOptions{
|
|
KeyPath: keyPath,
|
|
Posture: escrow.Posture(cfg.Escrow.Posture),
|
|
WantOfflineCopy: offline,
|
|
WantPaperkey: paperkey,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] escrow create:", err)
|
|
return 1
|
|
}
|
|
|
|
// Surface R EXACTLY ONCE — to stdout, with a write-it-down banner. Never logged/persisted.
|
|
fmt.Println()
|
|
fmt.Println(" ┌──────────────────────────────────────────────────────────────────────┐")
|
|
fmt.Println(" │ RECOVERY CODE — write it down now. It is shown ONCE and never stored. │")
|
|
fmt.Println(" │ Without it your offsite backups are unrecoverable, by anyone. │")
|
|
fmt.Println(" └──────────────────────────────────────────────────────────────────────┘")
|
|
fmt.Println(" " + R)
|
|
fmt.Println()
|
|
R = "" // drop our reference promptly
|
|
|
|
fmt.Printf(" blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n",
|
|
len(res.Blob), res.KeyFingerprint, res.Posture, res.EntropyBits)
|
|
fmt.Println(" self-verify: the blob unwraps back to the key with R (recoverability confirmed)")
|
|
|
|
if offline && len(res.OfflineCopy) > 0 {
|
|
fmt.Println(" --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---")
|
|
fmt.Println(base64.StdEncoding.EncodeToString(res.OfflineCopy))
|
|
}
|
|
if paperkey && res.Paperkey != "" {
|
|
fmt.Println(" --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---")
|
|
fmt.Println(res.Paperkey)
|
|
}
|
|
|
|
if upload {
|
|
if err := uploadEscrowBlob(ctx, cfg, res); err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] upload escrow to hub:", err)
|
|
return 1
|
|
}
|
|
fmt.Println(" uploaded the opaque blob to the hub (host record); the hub cannot open it")
|
|
}
|
|
fmt.Println("=== selftest=escrow-create OK ===")
|
|
return 0
|
|
}
|
|
|
|
// escrowUploadRequest is the agent→hub wire shape for the opaque escrow blob. MUST stay in lockstep
|
|
// with the hub's ingest struct (felhom-hub api.escrowUploadRequest). The hub stores the bytes and
|
|
// never decrypts them.
|
|
type escrowUploadRequest struct {
|
|
BlobB64 string `json:"blob_b64"` // base64 of the opaque R-wrapped blob (ciphertext)
|
|
KeyFingerprint string `json:"key_fingerprint"` // for operator display only
|
|
Posture string `json:"posture"` // e.g. "zero_knowledge"
|
|
CreatedAt string `json:"created_at"` // RFC3339
|
|
}
|
|
|
|
// uploadEscrowBlob PUTs the opaque blob to the hub, authed with the per-host key.
|
|
func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateResult) error {
|
|
if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
|
|
return fmt.Errorf("hub not configured (url/host_id/api_key)")
|
|
}
|
|
body, _ := json.Marshal(escrowUploadRequest{
|
|
BlobB64: base64.StdEncoding.EncodeToString(res.Blob),
|
|
KeyFingerprint: res.KeyFingerprint,
|
|
Posture: string(res.Posture),
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
})
|
|
url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Hub.APIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
|
return fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
|
|
// path), then lists + prints the resulting PBSSnapshot records (verify-state included).
|
|
// Standalone on the host. Covers the runbook's (c) verify and (d) list.
|
|
func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
|
|
return 1
|
|
}
|
|
px, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
|
return 1
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version)
|
|
targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, " [FAIL] discover pbs storages:", err)
|
|
return 1
|
|
}
|
|
if len(targets) == 0 {
|
|
fmt.Println(" no pbs storages configured on this host")
|
|
return 0
|
|
}
|
|
store := pbs.NewSnapshotStore()
|
|
loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
|
|
Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil },
|
|
Store: store,
|
|
Logger: logger,
|
|
})
|
|
// One synchronous verify+list pass over all datastores.
|
|
loop.RunOnce(ctx)
|
|
snaps := store.PBSSnapshots(ctx)
|
|
printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps)
|
|
failed := 0
|
|
for _, s := range snaps {
|
|
if s.VerifyState == pbs.VerifyFailed {
|
|
failed++
|
|
}
|
|
}
|
|
if failed > 0 {
|
|
fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed)
|
|
return 1
|
|
}
|
|
fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets))
|
|
return 0
|
|
}
|
|
|
|
// printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
|
|
func printJSON(label string, v any) {
|
|
if b, err := json.MarshalIndent(v, " ", " "); err == nil {
|
|
fmt.Printf(" --- %s ---\n %s\n", label, string(b))
|
|
}
|
|
}
|
|
|
|
// runSelftestRead loads config, builds the API client, and runs the read-only
|
|
// queries against the live host, printing a short health report. It mutates
|
|
// nothing. Missing/invalid config is reported cleanly (no panic).
|
|
func runSelftestRead(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
|
|
return 1
|
|
}
|
|
logger.Info("selftest (read-only) starting", "config", fmt.Sprintf("%+v", cfg.Redacted().Proxmox))
|
|
|
|
client, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: client init:", err)
|
|
return 1
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
fmt.Printf("=== felhom-agent %s selftest (read-only) ===\n", version)
|
|
fmt.Printf("endpoint : %s node=%s\n", cfg.Proxmox.Endpoint, cfg.Proxmox.Node)
|
|
|
|
fail := 0
|
|
report := func(label string, err error) bool {
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-14s %v\n", label, err)
|
|
fail++
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
if v, err := client.Version(ctx); report("version", err) {
|
|
fmt.Printf(" [ ok ] %-14s PVE %s (release %s)\n", "version", v.Version, v.Release)
|
|
}
|
|
if nodes, err := client.Nodes(ctx); report("nodes", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d node(s)\n", "nodes", len(nodes))
|
|
for _, n := range nodes {
|
|
marker := " "
|
|
if n.Node == cfg.Proxmox.Node {
|
|
marker = "* "
|
|
}
|
|
fmt.Printf(" %s%s status=%s fp=%s…\n", marker, n.Node, n.Status, head(n.SSLFingerprint, 17))
|
|
}
|
|
}
|
|
if s, err := client.NodeStatus(ctx); report("node status", err) {
|
|
fmt.Printf(" [ ok ] %-14s up %s, load %v, mem %s/%s, root %s/%s\n", "node status",
|
|
dur(s.Uptime), s.LoadAvg,
|
|
gib(s.Memory.Used), gib(s.Memory.Total), gib(s.RootFS.Used), gib(s.RootFS.Total))
|
|
}
|
|
if gs, err := client.ListLXC(ctx); report("list lxc", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d guest(s)\n", "list lxc", len(gs))
|
|
for _, g := range gs {
|
|
fmt.Printf(" - %d %q status=%s\n", g.VMID, g.Name, g.Status)
|
|
}
|
|
}
|
|
if ss, err := client.NodeStorage(ctx); report("storage", err) {
|
|
fmt.Printf(" [ ok ] %-14s %d store(s)\n", "storage", len(ss))
|
|
for _, s := range ss {
|
|
fmt.Printf(" - %-10s type=%-8s content=%s used=%s/%s\n",
|
|
s.Storage, s.Type, s.Content, gib(s.Used), gib(s.Total))
|
|
}
|
|
}
|
|
|
|
if fail > 0 {
|
|
fmt.Printf("=== selftest FAILED (%d check(s)) ===\n", fail)
|
|
return 1
|
|
}
|
|
fmt.Println("=== selftest OK ===")
|
|
return 0
|
|
}
|
|
|
|
// runSelftestTask exercises WaitTask on a reversible op against -vmid: snapshot ->
|
|
// rollback -> delete-snapshot. Explicitly gated; never runs under bare --selftest.
|
|
func runSelftestTask(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: not configured:", err)
|
|
return 1
|
|
}
|
|
if vmid == 0 {
|
|
fmt.Fprintln(os.Stderr, "selftest=task requires -vmid N (a guest safe to snapshot/rollback)")
|
|
return 2
|
|
}
|
|
client, err := newProxmoxClient(cfg)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "selftest: client init:", err)
|
|
return 1
|
|
}
|
|
|
|
// Ctrl-C aborts the wait cleanly.
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
const snap = "felhom-selftest"
|
|
steps := []struct {
|
|
name string
|
|
do func() (string, error)
|
|
}{
|
|
{"snapshot", func() (string, error) { return client.Snapshot(ctx, vmid, snap, "felhom-agent selftest") }},
|
|
{"rollback", func() (string, error) { return client.Rollback(ctx, vmid, snap) }},
|
|
{"delete-snapshot", func() (string, error) { return client.DeleteSnapshot(ctx, vmid, snap) }},
|
|
}
|
|
fmt.Printf("=== felhom-agent selftest=task (vmid %d, snapshot %q) ===\n", vmid, snap)
|
|
for _, st := range steps {
|
|
upid, err := st.do()
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", st.name, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" .... %-16s upid=%s\n", st.name, upid)
|
|
status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", st.name, err)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s exitstatus=%s\n", st.name, status.ExitStatus)
|
|
}
|
|
|
|
// Reversible SetConfig exercise: this is the first live use of the VM.Config.*
|
|
// privilege cluster. It round-trips the cosmetic `description` field (no runtime
|
|
// effect, fully reversible) to prove SetConfig works under the scoped token
|
|
// before slice 4's reconcile is built on top of it.
|
|
if rc := selftestSetConfig(ctx, client, vmid); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
fmt.Println("=== selftest=task OK ===")
|
|
return 0
|
|
}
|
|
|
|
// selftestSetConfig performs a reversible write+revert of the LXC `description`
|
|
// field on vmid and asserts both land. Returns 0 on success, non-zero (with a
|
|
// printed [FAIL] line) on any failure — the caller stops on a non-zero return.
|
|
func selftestSetConfig(ctx context.Context, client *proxmox.Client, vmid int) int {
|
|
// 1. Read current state; capture the original description (may be absent).
|
|
cfg, err := client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig: %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
origDesc, origPresent, err := extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description: %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
// PVE normalizes `description` by appending a trailing newline on read, so all
|
|
// comparisons here use the shared reconcile.NormDescription (strip trailing
|
|
// newlines) and restores write the normalized original — otherwise an exact-match
|
|
// check sees false drift. Same helper the slice-4 reconciler uses to normalize
|
|
// description, so the quirk has one source of truth.
|
|
origDesc = reconcile.NormDescription(origDesc)
|
|
|
|
// 2. Write the marker.
|
|
marker := "felhom-selftest " + time.Now().UTC().Format(time.RFC3339)
|
|
if rc := applySetConfig(ctx, client, vmid, "setconfig", map[string]string{"description": marker}); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
// 3. Verify the marker landed.
|
|
cfg, err = client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig (verify): %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
got, present, err := extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description (verify): %v\n", "setconfig", err)
|
|
return 1
|
|
}
|
|
if !present || reconcile.NormDescription(got) != marker {
|
|
fmt.Printf(" [FAIL] %-16s write did not land: present=%v got=%q want=%q\n", "setconfig", present, got, marker)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s description verified == marker\n", "verify-write")
|
|
|
|
// 4. Restore the original value (or clear it if it was absent originally).
|
|
var revert map[string]string
|
|
if origPresent {
|
|
revert = map[string]string{"description": origDesc}
|
|
} else {
|
|
revert = map[string]string{"delete": "description"}
|
|
}
|
|
if rc := applySetConfig(ctx, client, vmid, "setconfig-revert", revert); rc != 0 {
|
|
return rc
|
|
}
|
|
|
|
// 5. Confirm the restore.
|
|
cfg, err = client.GuestConfig(ctx, vmid)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s GuestConfig (revert verify): %v\n", "setconfig-revert", err)
|
|
return 1
|
|
}
|
|
got, present, err = extraString(cfg, "description")
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s decode description (revert verify): %v\n", "setconfig-revert", err)
|
|
return 1
|
|
}
|
|
if origPresent {
|
|
if !present || reconcile.NormDescription(got) != origDesc {
|
|
fmt.Printf(" [FAIL] %-16s revert did not restore: present=%v got=%q want=%q\n", "setconfig-revert", present, reconcile.NormDescription(got), origDesc)
|
|
return 1
|
|
}
|
|
} else if present {
|
|
fmt.Printf(" [FAIL] %-16s revert did not clear: still present got=%q\n", "setconfig-revert", got)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s description restored to original\n", "verify-revert")
|
|
return 0
|
|
}
|
|
|
|
// applySetConfig runs one SetConfig and asserts success, handling PVE's dual-mode
|
|
// return: a UPID means async (WaitTask + assert exitstatus OK); an empty string
|
|
// means PVE applied it synchronously (not an error — phase1-2/mutate.go contract).
|
|
func applySetConfig(ctx context.Context, client *proxmox.Client, vmid int, step string, params map[string]string) int {
|
|
upid, err := client.SetConfig(ctx, vmid, params)
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %v\n", step, err)
|
|
return 1
|
|
}
|
|
if upid == "" {
|
|
// Synchronous path: empty UPID is a clean success.
|
|
fmt.Printf(" [ ok ] %-16s synchronous exitstatus=OK\n", step)
|
|
return 0
|
|
}
|
|
status, err := client.WaitTask(ctx, upid, proxmox.WaitOptions{})
|
|
if err != nil {
|
|
fmt.Printf(" [FAIL] %-16s %s %v\n", step, upid, err)
|
|
return 1
|
|
}
|
|
if status.ExitStatus != "OK" {
|
|
fmt.Printf(" [FAIL] %-16s %s exitstatus=%s\n", step, upid, status.ExitStatus)
|
|
return 1
|
|
}
|
|
fmt.Printf(" [ ok ] %-16s %s exitstatus=%s\n", step, upid, status.ExitStatus)
|
|
return 0
|
|
}
|
|
|
|
// extraString reads a string-valued key from GuestConfig.Extra (raw JSON). It
|
|
// returns ("", false, nil) when the key is absent, and decodes the JSON string
|
|
// otherwise.
|
|
func extraString(cfg proxmox.GuestConfig, key string) (string, bool, error) {
|
|
raw, ok := cfg.Extra[key]
|
|
if !ok || len(raw) == 0 {
|
|
return "", false, nil
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
return "", false, err
|
|
}
|
|
return s, true, nil
|
|
}
|
|
|
|
// --- small helpers / flag type ---
|
|
|
|
func envOr(key, def string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|
|
|
|
func head(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n]
|
|
}
|
|
|
|
func dur(seconds int64) string { return (time.Duration(seconds) * time.Second).String() }
|
|
|
|
func gib(bytes int64) string { return fmt.Sprintf("%.1fGiB", float64(bytes)/(1<<30)) }
|
|
|
|
// selftestFlag is a flag.Value that also satisfies IsBoolFlag, so `--selftest`
|
|
// works bare (read-only) and `--selftest=task` / `--selftest=read` set the mode.
|
|
type selftestFlag struct{ mode string }
|
|
|
|
func (f *selftestFlag) String() string { return f.mode }
|
|
func (f *selftestFlag) IsBoolFlag() bool { return true }
|
|
func (f *selftestFlag) Set(v string) error {
|
|
switch v {
|
|
case "true", "", "read":
|
|
f.mode = "read"
|
|
case "task":
|
|
f.mode = "task"
|
|
case "hub":
|
|
f.mode = "hub"
|
|
case "storage":
|
|
f.mode = "storage"
|
|
case "backup":
|
|
f.mode = "backup"
|
|
case "restore-test":
|
|
f.mode = "restore-test"
|
|
case "pbs-verify":
|
|
f.mode = "pbs-verify"
|
|
case "bring-up":
|
|
f.mode = "bring-up"
|
|
case "provision":
|
|
f.mode = "provision"
|
|
case "escrow-create":
|
|
f.mode = "escrow-create"
|
|
default:
|
|
return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|provision|escrow-create)", v)
|
|
}
|
|
return nil
|
|
}
|