slice 8A (agent half): local-API server + provisioning back-half (v0.10.0)

internal/localapi: per-guest local-API server (doc 03 §6) — 7 self-scoped endpoints, hashed per-guest token store, persisted self-signed leaf with stable SHA-256 pin, optional 6th daemon goroutine. internal/provision: back-half — mint token, render bootstrap.json (no registry cred), write 0600, chown 100000:100000, attach pct-set bind mount (host-side, F3, no pct exec). --selftest=provision. build-golden.sh bakes the controller image + bootstrap unit. sudoers FELHOM_PROVISION; firewall narrowing artifact. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-10 09:47:42 +02:00
parent fae11020a5
commit 3fecf4c713
18 changed files with 2203 additions and 11 deletions
@@ -15,6 +15,7 @@ import (
 	"flag"
 	"fmt"
 	"log/slog"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -28,8 +29,10 @@ import (
 	"gitea.dooplex.hu/admin/felhom-agent/internal/config"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
 	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
@@ -37,7 +40,7 @@ import (

 // version is the agent version. Overridable at build time with
 // -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
-var version = "0.9.0"
+var version = "0.10.0"

 func main() {
 	var (
@@ -53,10 +56,14 @@ func main() {
 		paperkey    bool
 		offline     bool
 		upload      bool
+		custID      string
+		custDomain  string
+		custName    string
+		custEmail   string
 		showVersion bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; tears down unless -keep)")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-customer-domain; keeps the guest)")
 	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
 	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
 	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
@@ -67,6 +74,10 @@ func main() {
 	flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
 	flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
 	flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
+	flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id to seed into the guest's bootstrap")
+	flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: the customer domain to seed")
+	flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: the customer display name to seed (optional)")
+	flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: the customer email to seed (optional)")
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.Parse()

@@ -106,6 +117,11 @@ func main() {
 		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
 	case "bring-up":
 		os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep))
+	case "provision":
+		os.Exit(runSelftestProvision(context.Background(), cfg, logger, provisionArgs{
+			archive: archive, vmid: vmid, hostname: hostname,
+			customer: provision.DocCustomer{ID: custID, Domain: custDomain, Name: custName, Email: custEmail},
+		}))
 	case "escrow-create":
 		os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload))
 	}
@@ -315,18 +331,34 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 		Logger:  logger,
 	})

-	// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, and the
-	// PBS verify loop concurrently; any one returning ends the daemon (ctx cancel tears down rest).
-	errc := make(chan error, 5)
+	// Local API server (slice 8A, doc 03 §6): the per-guest authorization gate the in-guest
+	// controller calls over the bridge. Optional — runs only when local_api.enable + listen_addr
+	// are configured; a token-store or cert failure disables it WITHOUT killing the daemon (the
+	// host still reports/reconciles). The leaf is generated+persisted once so its pin is stable.
+	localServers := 0
+	var localTokens *localapi.TokenStore
+	localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, logger, &localTokens)
+	if localTokens != nil {
+		defer localTokens.Close()
+	}
+
+	// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
+	// verify loop, and (optionally) the local-API server concurrently; any one returning ends
+	// the daemon (ctx cancel tears down the rest).
+	errc := make(chan error, 6)
 	go func() { errc <- engine.Run(ctx, interval) }()
 	go func() { errc <- loop.Run(ctx) }()
 	go func() { errc <- watchdog.Run(ctx) }()
 	go func() { errc <- scheduler.Run(ctx) }()
 	go func() { errc <- pbsLoop.Run(ctx) }()
+	if localSrv != nil {
+		localServers = 1
+		go func() { errc <- localSrv.Run(ctx) }()
+	}

 	err = <-errc
-	stop()                   // tear down the siblings on the first exit
-	for i := 0; i < 4; i++ { // wait for the other four
+	stop()                                  // tear down the siblings on the first exit
+	for i := 0; i < 4+localServers; i++ {   // wait for the other goroutines
 		<-errc
 	}
 	if err != nil && err != context.Canceled {
@@ -425,6 +457,50 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
 	})
 }

+// buildLocalAPIServer constructs the per-guest local-API server (slice 8A, doc 03 §6) when
+// configured. It opens the durable hashed token store and ensures the persisted self-signed
+// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
+// daemon — the host still reports/reconciles; only the controller channel is unavailable until
+// fixed. The opened token store is returned via outTokens so the caller can Close it.
+func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
+	if !cfg.LocalAPI.Enabled() {
+		return nil
+	}
+	if err := cfg.LocalAPI.Validate(); err != nil {
+		logger.Warn("daemon: local-api disabled (config invalid)", "err", err)
+		return nil
+	}
+	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
+	if err != nil {
+		logger.Warn("daemon: local-api disabled (token store)", "err", err)
+		return nil
+	}
+	*outTokens = tokens
+	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
+	cert, fp, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
+	if err != nil {
+		logger.Warn("daemon: local-api disabled (leaf cert)", "err", err)
+		return nil
+	}
+	logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
+	runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger)
+	srv, err := localapi.NewServer(localapi.Options{
+		ListenAddr: cfg.LocalAPI.ListenAddr,
+		Cert:       cert,
+		Guests:     px,
+		Backups:    runner,
+		Store:      store,
+		Storage:    observer,
+		Tokens:     tokens,
+		Logger:     logger,
+	})
+	if err != nil {
+		logger.Warn("daemon: local-api disabled (server build)", "err", err)
+		return nil
+	}
+	return srv
+}
+
 // reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
 // configured nonce store (both are durable agent state), falling back to the standard
 // host state dir when the nonce store is unset.
@@ -814,6 +890,121 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
 	return 0
 }

+// provisionArgs bundles the --selftest=provision inputs.
+type provisionArgs struct {
+	archive  string
+	vmid     int
+	hostname string
+	customer provision.DocCustomer
+}
+
+// runSelftestProvision runs the FULL slice-8A provisioning chain on-demand: the slice-7 bring-up
+// FRONT half (provision mode, golden) + the slice-8A BACK half (mint per-guest token → render the
+// stable bootstrap.json → write 0600 → chown to the mapped guest-root → attach the read-only bind
+// mount). The guest is KEPT (the golden's baked controller-bootstrap unit then deploys the baked
+// controller from the mount). The per-guest token is NEVER printed (only its hash is persisted +
+// the 0600 file holds the plaintext).
+func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.Logger, a provisionArgs) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	if a.archive == "" || a.vmid <= 0 {
+		fmt.Fprintln(os.Stderr, "selftest=provision requires -archive <golden volid> and -vmid <N>")
+		return 2
+	}
+	if cfg.Backup.RestoreStorage == "" {
+		fmt.Fprintln(os.Stderr, "selftest=provision requires backup.restore_storage in config")
+		return 2
+	}
+	if a.customer.ID == "" || a.customer.Domain == "" {
+		fmt.Fprintln(os.Stderr, "selftest=provision requires -customer-id and -customer-domain (so the controller skips setup)")
+		return 2
+	}
+	if err := cfg.LocalAPI.Validate(); err != nil || !cfg.LocalAPI.Enabled() {
+		fmt.Fprintln(os.Stderr, "selftest=provision requires local_api.enable + local_api.listen_addr in config:", err)
+		return 2
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// The leaf fingerprint baked into the bootstrap must be the SAME leaf the daemon's local-API
+	// server serves — so use the configured (persisted) cert path. EnsureLeaf generates it once.
+	host, _, _ := net.SplitHostPort(cfg.LocalAPI.ListenAddr)
+	_, fingerprint, err := localapi.EnsureLeaf(cfg.LocalAPI.CertPath(), cfg.LocalAPI.KeyPath(), host)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest=provision: local-api leaf:", err)
+		return 1
+	}
+	tokens, err := localapi.OpenTokenStore(cfg.LocalAPI.TokenStorePath())
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest=provision: token store:", err)
+		return 1
+	}
+	defer tokens.Close()
+
+	// --- FRONT HALF: bring up the guest (provision mode) ---
+	queue := reconcile.NewQueue()
+	defer queue.Close()
+	var journal *reconcile.Journal
+	if jp := reconcileJournalPath(cfg); jp != "" {
+		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
+			if j, err := reconcile.OpenJournal(jp); err == nil {
+				journal = j
+				defer journal.Close()
+			}
+		}
+	}
+	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
+	engine := reconcile.NewEngine(reconcile.EngineOptions{
+		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
+	})
+
+	fmt.Printf("=== felhom-agent %s selftest=provision (vmid=%d customer=%s) ===\n", version, a.vmid, a.customer.ID)
+	engine.Recover(ctx)
+	fmt.Printf("  --- front half: bring-up (provision) %s → vmid %d ---\n", a.archive, a.vmid)
+	res := engine.RunBringUp(ctx, reconcile.BringUpSpec{
+		Mode: reconcile.ModeProvision, Archive: a.archive, VMID: a.vmid,
+		RestoreStorage: cfg.Backup.RestoreStorage, Hostname: a.hostname,
+	})
+	if res.Err != nil || !res.Pass {
+		fmt.Fprintf(os.Stderr, "  [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)
+		return 1
+	}
+	fmt.Printf("  [OK] front half: vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
+
+	// --- BACK HALF: mint token + populate the bootstrap config mount (host-side, no pct exec) ---
+	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
+	if mode == "" {
+		mode = proxmox.RunnerSudo
+	}
+	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
+	bh := provision.NewBackHalf(tokens, runner, "", logger)
+	fmt.Println("  --- back half: mint per-guest token + populate bootstrap config mount ---")
+	pres, err := bh.Provision(ctx, provision.Input{
+		VMID:        a.vmid,
+		Customer:    a.customer,
+		Hub:         provision.DocHub{URL: cfg.Hub.URL, APIKey: cfg.Hub.APIKey, HostID: cfg.Hub.HostID},
+		Endpoint:    cfg.LocalAPI.ListenAddr,
+		Fingerprint: fingerprint,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "  [FAIL] back-half provision (vmid %d): %v\n", a.vmid, err)
+		return 1
+	}
+	fmt.Printf("  [OK] back half: bootstrap mount %s → %s on vmid %d (host dir %s)\n",
+		pres.MountKey, pres.GuestPath, pres.VMID, pres.HostDir)
+	fmt.Printf("  local-api endpoint %s · leaf fp %s · token: minted (not printed)\n", cfg.LocalAPI.ListenAddr, fingerprint)
+	fmt.Printf("=== selftest=provision OK — guest %d provisioned + bootstrap-mounted (KEPT) ===\n", a.vmid)
+	fmt.Println("    next: the golden's baked controller-bootstrap unit deploys the controller from the mount on boot.")
+	return 0
+}
+
 // runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
 // wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
 // blob. R is surfaced to stdout EXACTLY ONCE (never to the logger/journald). With -upload it PUTs
@@ -1273,10 +1464,12 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "pbs-verify"
 	case "bring-up":
 		f.mode = "bring-up"
+	case "provision":
+		f.mode = "provision"
 	case "escrow-create":
 		f.mode = "escrow-create"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|escrow-create)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|provision|escrow-create)", v)
 	}
 	return nil
 }