v0.88.0: controller-driven escrow ceremony — --output=json machine mode (escrowCeremony extraction, text mode byte-identical), the ONE fixed argv (escrow.CeremonyArgs, shared by exec+manifest+FELHOM_ESCROW sudoers, pin-tested), localapi ceremony job (single-flight, 60s) + one-shot in-memory R claim (10min TTL, unclaimed_void) + preflight; escrow-ceremony capability (Critical, pbs_dr-gated)

2026-07-13 19:01:11 +02:00
parent adf7882f7d
commit 1c3a3ef9ad
11 changed files with 1196 additions and 52 deletions
@@ -156,6 +156,7 @@ func main() {
 		idBundlePath  string
 		directivePath string
 		swapImage     string
+		outputMode    string
 		showVersion   bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
@@ -189,6 +190,7 @@ func main() {
 	flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)")
 	flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)")
 	flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)")
+	flag.StringVar(&outputMode, "output", "text", "for --selftest=escrow-create: `text` (human, default — unchanged) | `json` (one machine-readable JSON object on stdout carrying the recovery code; every human line to stderr; the controller-driven ceremony's parse surface)")
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.Parse()

@@ -244,7 +246,7 @@ func main() {
 				SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB},
 		}))
 	case "escrow-create":
-		os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath))
+		os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath, outputMode))
 	case "escrow-consume":
 		os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest))
 	case "identity-consume":
@@ -722,7 +724,21 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 	jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
 	loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))

-	localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, logger, &localTokens)
+	// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
+	// the pbsdr manager is constructed further down; the closure reads drConfigured at call time
+	// (nil until then → preflight reports the tier not applied, which is the honest pre-wire answer).
+	escrowCeremonyCfg := &localapi.EscrowCeremonyConfig{
+		SudoPath:      cfg.Privileged.SudoPath,
+		PBSStorageID:  cfg.Escrow.PBSStorageID,
+		HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "",
+		DRConfigured: func() bool {
+			if drConfigured != nil {
+				return drConfigured()
+			}
+			return false
+		},
+	}
+	localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
 	if localTokens != nil {
 		defer localTokens.Close()
 	}
@@ -1071,7 +1087,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
 // leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
 // daemon — the host still reports/reconciles; only the controller channel is unavailable until
 // fixed. The opened token store is returned via outTokens so the caller can Close it.
-func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
+func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
 	if !cfg.LocalAPI.Enabled() {
 		return nil
 	}
@@ -1143,7 +1159,9 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
 		HostMetrics: collector,
 		HostID:      cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op
 		LogRing:     logRing,        // v0.83.0: GET /debug/logs — the always-DEBUG capture ring
-		Logger:      logger,
+		// Controller-driven escrow ceremony (v0.88.0): the customer wizard's agent half.
+		EscrowCeremony: escrowCeremony,
+		Logger:         logger,
 	})
 	if err != nil {
 		logger.Warn("daemon: local-api disabled (server build)", "err", err)
@@ -1798,22 +1816,58 @@ func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.L
 	return 0
 }

-// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
-// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
-// blob. R is surfaced to stdout EXACTLY ONCE (never to the logger/journald). With -upload it PUTs
-// the opaque blob to the hub. Enrollment-time, root-capable (reads the 0600 key).
-func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath string) int {
+// escrowCeremonyOpts carries the CLI switches into the shared ceremony core (v0.88.0 extraction —
+// the work is identical for both output modes; only the surfacing differs).
+type escrowCeremonyOpts struct {
+	storage            string
+	paperkey           bool
+	offline            bool
+	upload             bool
+	identityBundlePath string
+	directivePath      string
+}
+
+// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the
+// --output=json wire object with RecoveryCode left EMPTY (the shells place R themselves, once).
+type escrowCeremonyOutcome struct {
+	R           string
+	Sum         escrow.CeremonyOutput
+	OfflineCopy []byte // text-mode print block (opt-in b)
+	Paperkey    string // text-mode print block (opt-in a) — SECRET-adjacent
+	Posture     escrow.Posture
+	Storage     string // banner data
+	Identity    bool   // banner data
+}
+
+// escrowCeremonyErr classifies a ceremony failure so both output modes keep the pre-v0.88.0 exit
+// codes and stderr shapes: usage → exit 2; setup/create/upload → exit 1. kind "upload" means R
+// was already minted — the text shell still surfaces it before the failure (the customer must
+// receive R; the blob simply never reached the hub), exactly the pre-extraction print order.
+type escrowCeremonyErr struct {
+	kind string // "usage" | "setup" | "create" | "upload"
+	err  error
+}
+
+func (e *escrowCeremonyErr) Error() string { return e.err.Error() }
+
+// escrowCeremony is the shared ceremony core (slice 7 + 10D.1 + fork-4, extracted v0.88.0 for the
+// --output=json machine mode): resolve the key, assemble the identity bundle (WG key + staged
+// restic password auto-attach), Create (R + self-verified blob), wipe the staged secret, upload
+// when asked. It PRINTS NOTHING — the output-mode shells own every byte of stdout/stderr. R is
+// returned for the caller to surface exactly once; escrow.Create never logs it and neither do we.
+func escrowCeremony(ctx context.Context, cfg config.Config, logger *slog.Logger, opts escrowCeremonyOpts) (escrowCeremonyOutcome, *escrowCeremonyErr) {
+	var out escrowCeremonyOutcome
+	storage := opts.storage
 	if storage == "" {
 		storage = cfg.Escrow.PBSStorageID
 	}
 	if storage == "" {
-		fmt.Fprintln(os.Stderr, "selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")
-		return 2
+		return out, &escrowCeremonyErr{kind: "usage", err: fmt.Errorf("selftest=escrow-create requires -storage <pbs-storage-id> (or escrow.pbs_storage_id)")}
 	}
+	out.Storage = storage
 	keyPath := cfg.Backup.PBSEncKeyPath(storage)
 	if _, err := os.Stat(keyPath); err != nil {
-		fmt.Fprintf(os.Stderr, "selftest=escrow-create: PBS key for %q not found (%s): %v\n", storage, keyPath, err)
-		return 1
+		return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("PBS key for %q not found (%s): %v", storage, keyPath, err)}
 	}

 	// Slice 10D.1: optionally ALSO wrap the identity bundle under the same R, and carry the non-secret
@@ -1821,20 +1875,18 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
 	// non-secret (pbs repo/ns, expected fingerprint, tunnel id).
 	var identity *escrow.IdentityBundle
 	var directive json.RawMessage
-	if identityBundlePath != "" {
-		raw, err := os.ReadFile(identityBundlePath)
+	if opts.identityBundlePath != "" {
+		raw, err := os.ReadFile(opts.identityBundlePath)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "selftest=escrow-create: reading identity bundle %s: %v\n", identityBundlePath, err)
-			return 1
+			return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("reading identity bundle %s: %v", opts.identityBundlePath, err)}
 		}
 		var b escrow.IdentityBundle
 		if err := json.Unmarshal(raw, &b); err != nil {
-			fmt.Fprintf(os.Stderr, "selftest=escrow-create: identity bundle is not valid JSON {tunnel_token,pbs_token}: %v\n", err)
-			return 1
+			return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("identity bundle is not valid JSON {tunnel_token,pbs_token}: %v", err)}
 		}
 		identity = &b
-		if directivePath != "" {
-			if d, err := os.ReadFile(directivePath); err == nil && json.Valid(d) {
+		if opts.directivePath != "" {
+			if d, err := os.ReadFile(opts.directivePath); err == nil && json.Valid(d) {
 				directive = d
 			}
 		}
@@ -1850,8 +1902,7 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
 		}
 		attached, err := escrow.AttachWGKey(probe, wgKeyPath)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", err)
-			return 1
+			return out, &escrowCeremonyErr{kind: "setup", err: err}
 		}
 		if attached {
 			identity = probe
@@ -1870,8 +1921,7 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
 		}
 		attached, err := escrow.AttachResticPassword(probe, escrow.StagedResticPasswordPath())
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", err)
-			return 1
+			return out, &escrowCeremonyErr{kind: "setup", err: err}
 		}
 		if attached {
 			identity = probe
@@ -1882,21 +1932,20 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
 			logger.Info("escrow: identity bundle: +restic_repo_password")
 		}
 	}
+	out.Identity = identity != nil

-	fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, storage, escrow.DefaultPosture, identity != nil)
-	// NB: nothing about R is logged. The logger never sees R; only stdout does, once.
+	// NB: nothing about R is logged. The logger never sees R; only the shells surface it, once.
 	logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath, "with_identity", identity != nil)

 	R, res, err := escrow.Create(ctx, escrow.CreateOptions{
 		KeyPath:         keyPath,
 		Posture:         escrow.Posture(cfg.Escrow.Posture),
-		WantOfflineCopy: offline,
-		WantPaperkey:    paperkey,
+		WantOfflineCopy: opts.offline,
+		WantPaperkey:    opts.paperkey,
 		IdentityBundle:  identity,
 	})
 	if err != nil {
-		fmt.Fprintln(os.Stderr, "  [FAIL] escrow create:", err)
-		return 1
+		return out, &escrowCeremonyErr{kind: "create", err: err}
 	}
 	// fork-4: the staged restic password is now sealed inside the R-wrapped blob — wipe the transient
 	// 0600 staging file so it never lingers on disk (field name only; a wipe failure is a loud warn).
@@ -1906,37 +1955,149 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
 		}
 	}

-	// Surface R EXACTLY ONCE — to stdout, with a write-it-down banner. Never logged/persisted.
+	out.R = R
+	out.Posture = res.Posture
+	out.OfflineCopy = res.OfflineCopy
+	out.Paperkey = res.Paperkey
+	out.Sum = escrow.CeremonyOutput{
+		Version:           escrow.CeremonyOutputVersion,
+		KeyFingerprint:    res.KeyFingerprint,
+		EntropyBits:       res.EntropyBits,
+		BlobBytes:         len(res.Blob),
+		IdentityBlobBytes: len(res.IdentityBlob),
+		ResticPwSealed:    resticStaged,
+	}
+	if opts.upload {
+		if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil {
+			// R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets
+			// the text shell keep the pre-extraction order (R surfaced, THEN the failure).
+			return out, &escrowCeremonyErr{kind: "upload", err: err}
+		}
+		out.Sum.Uploaded = true
+	}
+	return out, nil
+}
+
+// printEscrowTextBanner prints the text-mode header line (shared by the success and the
+// post-banner failure paths so the pre-extraction output stays byte-identical — including the
+// posture: the historical banner always printed the DEFAULT posture, not the result's).
+func printEscrowTextBanner(out escrowCeremonyOutcome) {
+	fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, out.Storage, escrow.DefaultPosture, out.Identity)
+}
+
+// printEscrowTextRBlock surfaces R EXACTLY ONCE — to stdout, with the write-it-down banner —
+// followed by the non-secret blob facts and the opt-in print blocks. Never logged/persisted.
+func printEscrowTextRBlock(out *escrowCeremonyOutcome) {
 	fmt.Println()
 	fmt.Println("  ┌──────────────────────────────────────────────────────────────────────┐")
 	fmt.Println("  │  RECOVERY CODE — write it down now. It is shown ONCE and never stored.  │")
 	fmt.Println("  │  Without it your offsite backups are unrecoverable, by anyone.          │")
 	fmt.Println("  └──────────────────────────────────────────────────────────────────────┘")
-	fmt.Println("     " + R)
+	fmt.Println("     " + out.R)
 	fmt.Println()
-	R = "" // drop our reference promptly
+	out.R = "" // drop our reference promptly

 	fmt.Printf("  blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n",
-		len(res.Blob), res.KeyFingerprint, res.Posture, res.EntropyBits)
+		out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Posture, out.Sum.EntropyBits)
 	fmt.Println("  self-verify: the blob unwraps back to the key with R (recoverability confirmed)")

-	if offline && len(res.OfflineCopy) > 0 {
+	if len(out.OfflineCopy) > 0 {
 		fmt.Println("  --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---")
-		fmt.Println(base64.StdEncoding.EncodeToString(res.OfflineCopy))
+		fmt.Println(base64.StdEncoding.EncodeToString(out.OfflineCopy))
 	}
-	if paperkey && res.Paperkey != "" {
+	if out.Paperkey != "" {
 		fmt.Println("  --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---")
-		fmt.Println(res.Paperkey)
+		fmt.Println(out.Paperkey)
 	}

-	if len(res.IdentityBlob) > 0 {
-		fmt.Printf("  identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", len(res.IdentityBlob))
+	if out.Sum.IdentityBlobBytes > 0 {
+		fmt.Printf("  identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", out.Sum.IdentityBlobBytes)
 	}
-	if upload {
-		if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil {
-			fmt.Fprintln(os.Stderr, "  [FAIL] upload escrow to hub:", err)
+}
+
+// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R,
+// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque
+// blob. With -upload it PUTs the opaque blob to the hub. Enrollment-time, root-capable (reads the
+// 0600 key). Output modes (v0.88.0):
+//   - text (default): the historical human output, byte-identical to pre-v0.88.0 — R to stdout
+//     EXACTLY ONCE inside the write-it-down banner (never to the logger/journald).
+//   - json: ONE machine-readable JSON object on stdout (escrow.CeremonyOutput — carries R) and
+//     NOTHING else there; every human/info line goes to stderr; failures exit non-zero with no
+//     partial JSON. This is the controller-driven ceremony's parse surface (spike §2.3: the text
+//     banner is positionally brittle).
+func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath, outputMode string) int {
+	switch outputMode {
+	case "", "text", "json":
+	default:
+		fmt.Fprintf(os.Stderr, "selftest=escrow-create: unknown -output %q (text|json)\n", outputMode)
+		return 2
+	}
+	jsonMode := outputMode == "json"
+	if jsonMode && (offline || paperkey) {
+		// The opt-in print blocks are PRINT-oriented (base64 blob / paperkey text on stdout) —
+		// in json mode stdout carries exactly one JSON object, so refuse loudly instead of
+		// silently dropping what the caller asked for.
+		fmt.Fprintln(os.Stderr, "selftest=escrow-create: -offline/-paperkey are text-mode only (their output is print-oriented)")
+		return 2
+	}
+
+	out, cerr := escrowCeremony(ctx, cfg, logger, escrowCeremonyOpts{
+		storage: storage, paperkey: paperkey, offline: offline, upload: upload,
+		identityBundlePath: identityBundlePath, directivePath: directivePath,
+	})
+	if cerr != nil {
+		switch cerr.kind {
+		case "usage":
+			fmt.Fprintln(os.Stderr, cerr.err)
+			return 2
+		case "setup":
+			fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", cerr.err)
+			return 1
+		case "create":
+			if !jsonMode {
+				printEscrowTextBanner(out)
+			}
+			fmt.Fprintln(os.Stderr, "  [FAIL] escrow create:", cerr.err)
+			return 1
+		default: // "upload" — R was minted; text mode still surfaces it (pre-extraction order)
+			if !jsonMode {
+				printEscrowTextBanner(out)
+				printEscrowTextRBlock(&out)
+			}
+			out.R = ""
+			fmt.Fprintln(os.Stderr, "  [FAIL] upload escrow to hub:", cerr.err)
 			return 1
 		}
+	}
+
+	if jsonMode {
+		// Human/info lines → stderr (the job runner's diagnostics tail); the ONE JSON object with
+		// R → stdout. The consumer (localapi ceremony job) extracts R and zeroes its buffers.
+		fmt.Fprintf(os.Stderr, "=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v output=json) ===\n", version, out.Storage, out.Posture, out.Identity)
+		fmt.Fprintf(os.Stderr, "  blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · ~%.0f bits R · self-verify OK\n",
+			out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Sum.EntropyBits)
+		if out.Sum.IdentityBlobBytes > 0 {
+			fmt.Fprintf(os.Stderr, "  identity escrow: %d bytes · restic_pw_sealed=%v\n", out.Sum.IdentityBlobBytes, out.Sum.ResticPwSealed)
+		}
+		if out.Sum.Uploaded {
+			fmt.Fprintln(os.Stderr, "  uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
+		}
+		wire := out.Sum
+		wire.RecoveryCode = out.R
+		if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil {
+			fmt.Fprintf(os.Stderr, "selftest=escrow-create: emitting JSON: %v\n", err)
+			return 1
+		}
+		// Best-effort scrub: drop every R reference promptly. Go's GC may retain stale copies of
+		// the string backing array — this shrinks the window, it cannot guarantee erasure.
+		wire.RecoveryCode = ""
+		out.R = ""
+		return 0
+	}
+
+	printEscrowTextBanner(out)
+	printEscrowTextRBlock(&out)
+	if out.Sum.Uploaded {
 		fmt.Println("  uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
 	}
 	fmt.Println("=== selftest=escrow-create OK ===")