From 1c3a3ef9add1abd349823daea39b2045d6bd934d Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 13 Jul 2026 19:01:11 +0200 Subject: [PATCH] =?UTF-8?q?v0.88.0:=20controller-driven=20escrow=20ceremon?= =?UTF-8?q?y=20=E2=80=94=20--output=3Djson=20machine=20mode=20(escrowCerem?= =?UTF-8?q?ony=20extraction,=20text=20mode=20byte-identical),=20the=20ONE?= =?UTF-8?q?=20fixed=20argv=20(escrow.CeremonyArgs,=20shared=20by=20exec+ma?= =?UTF-8?q?nifest+FELHOM=5FESCROW=20sudoers,=20pin-tested),=20localapi=20c?= =?UTF-8?q?eremony=20job=20(single-flight,=2060s)=20+=20one-shot=20in-memo?= =?UTF-8?q?ry=20R=20claim=20(10min=20TTL,=20unclaimed=5Fvoid)=20+=20prefli?= =?UTF-8?q?ght;=20escrow-ceremony=20capability=20(Critical,=20pbs=5Fdr-gat?= =?UTF-8?q?ed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 36 ++ REUSE.md | 2 + cmd/felhom-agent/main.go | 249 ++++++++++--- configs/felhom-agent.sudoers | 12 +- internal/capability/manifest.go | 15 +- internal/capability/manifest_test.go | 47 +++ internal/capability/probe_test.go | 16 +- internal/escrow/ceremony.go | 49 +++ internal/localapi/escrow_ceremony.go | 421 ++++++++++++++++++++++ internal/localapi/escrow_ceremony_test.go | 360 ++++++++++++++++++ internal/localapi/server.go | 41 +++ 11 files changed, 1196 insertions(+), 52 deletions(-) create mode 100644 internal/escrow/ceremony.go create mode 100644 internal/localapi/escrow_ceremony.go create mode 100644 internal/localapi/escrow_ceremony_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 883b541..b7de577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,39 @@ +## v0.88.0 — controller-driven escrow ceremony: --output=json + localapi job + one-shot R claim (2026-07-13) + +The agent half of the customer-facing recovery-code wizard (controller v0.127.0; every mechanism +validated by felhom.eu SPIKE-controller-escrow-2026-07-13 — PTY-under-no-TTY, fixed-argv sudoers +5/5 refusals, R pipe round-trip, env_reset, 2.3–2.4 s timings). Operator ruling F1 (2026-07-13): +R transiting the CF tunnel once at display is an accepted, documented risk (threat-model paragraph +in RUNBOOK-escrow-ceremony.md). + +- **`--output=json` machine mode** (`cmd/felhom-agent/main.go`): `runSelftestEscrowCreate`'s body + extracted into the shared `escrowCeremony()` core; text mode stays BYTE-IDENTICAL (banner, R + block, exit codes 0/1/2, upload-fail-after-R order). json mode emits ONE + `escrow.CeremonyOutput` object on stdout (version 1: recovery_code, key_fingerprint, + entropy_bits, blob/identity sizes, restic_pw_sealed, uploaded), every human line to stderr, no + partial JSON on failure; `--offline`/`--paperkey` are refused in json mode (print-oriented). +- **The ONE fixed argv** (`internal/escrow/ceremony.go`): `escrow.CeremonyBinary` + + `escrow.CeremonyArgs()` — the single source shared by the localapi exec, the capability + manifest entry, and (byte-identically) the new `FELHOM_ESCROW` sudoers alias + (`configs/felhom-agent.sudoers`). `TestEscrowCeremonyArgvPinned` + + `TestManifestCoveredBySudoers` transitively lock runner == manifest == sudoers; never build the + argv with flag helpers, never normalize `--`→`-` (spike §2.2). +- **localapi ceremony endpoints** (`internal/localapi/escrow_ceremony.go`, `withGuest`-wrapped): + `POST /escrow/ceremony` (single-flight 409; detached job, 60 s timeout; `sudo -n` + the fixed + argv; stdout parsed then zeroed — SECRET-BEARING, never logged), `GET /escrow/ceremony/status` + (non-secret summary + stderr-tail failure detail ≤500; R structurally absent from the job + struct), `POST /escrow/ceremony/claim` (ONE-SHOT: 200 `{recovery_code}` once → holder zeroed; + 410 on re-claim; **TTL 10 min** → `unclaimed_void`, active AfterFunc belt + lazy check), + `GET /escrow/preflight` (storage id, DR tier, age, hub target, staged-secret informational, + `sudo -n -l` grant list-probe). Crash-safety is IN-MEMORY BY DESIGN — an agent restart loses R + safely (re-run supersedes); no journal, deliberately. +- **Capability** `escrow-ceremony` (Critical, `GatedBy: pbs_dr` EXPLICIT — non-pbsdr name by + decision): list-mode probe of the shared argv; inactive (never red) while the DR tier is off. +- Tests: one-shot claim + double-claim 410, TTL void + zeroed holder, R-substring absent from + every status/snapshot payload (incl. the serialized job struct), single-flight, supersede on + re-run, failure taxonomy (exit/unparseable/version), preflight truth table, argv pin. §10 + red-proofs demonstrated (see felhom.eu REPORT). + ## v0.87.0 — SystemDisks device-mapper walk: legacy-boot hosts get a working drive wizard (IA finding 2, MEDIUM) (2026-07-13) On a legacy-boot PVE (LVM root, no mounted ESP) `SystemDisks` resolved NOTHING — `wholeDiskOf` diff --git a/REUSE.md b/REUSE.md index 2632cf5..120a1d7 100644 --- a/REUSE.md +++ b/REUSE.md @@ -91,6 +91,8 @@ | `reconcile.Queue.Submit` | internal/reconcile/queue.go | `Submit(vmid, fn) <-chan error` | per-guest serialization of ALL mutations | Same vmid strictly FIFO; lanes parallel across guests | | `Engine.RunSignedJob` | internal/reconcile/job.go | `RunSignedJob(ctx, intent, signed, exec) JobResult` | executing a gated destructive job | Idempotency by nonce; journaled | | `escrow.Create` | internal/escrow/escrow.go | `Create(ctx, CreateOptions) (CreateResult, R, error)` | PBS-key escrow (zero-knowledge) | Recovery code returned SEPARATELY from the result (anti-log); self-verifies recoverability | +| `escrow.CeremonyBinary` / `CeremonyArgs()` / `CeremonyOutput` | internal/escrow/ceremony.go | the ONE fixed sudo self-invocation argv + the `--output=json` wire object (v1) | controller-driven ceremony (v0.88.0) | SINGLE SOURCE shared by the localapi exec, the capability manifest entry, and (byte-identically) the FELHOM_ESCROW sudoers line — `TestEscrowCeremonyArgvPinned` + `TestManifestCoveredBySudoers` lock all three. Never flag-helpers, never `--`→`-` (spike §2.2) | +| localapi escrow ceremony job | internal/localapi/escrow_ceremony.go | `POST /escrow/ceremony` + status + ONE-SHOT claim + preflight | the wizard's agent half | R lives ONLY in `Server.escrowR` (NEVER the job struct — snapshots must be structurally R-free); zeroed on claim/supersede/10-min TTL (`unclaimed_void`); in-memory BY DESIGN (restart loses R safely; re-run supersedes); subprocess stdout is SECRET-BEARING → parsed then zeroed, never logged | ## 2. Canonical patterns (copy structure from THE named file) diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index e465909..97a83db 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -156,6 +156,7 @@ func main() { idBundlePath string directivePath string swapImage string + outputMode string showVersion bool ) flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)") @@ -189,6 +190,7 @@ func main() { flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)") flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)") flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)") + flag.StringVar(&outputMode, "output", "text", "for --selftest=escrow-create: `text` (human, default — unchanged) | `json` (one machine-readable JSON object on stdout carrying the recovery code; every human line to stderr; the controller-driven ceremony's parse surface)") flag.BoolVar(&showVersion, "version", false, "print version and exit") flag.Parse() @@ -244,7 +246,7 @@ func main() { SysDataGrowGB: sysDataGrow, SysDataMount: sysDataMount, Cores: cores, MemoryMB: memoryMB}, })) case "escrow-create": - os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath)) + os.Exit(runSelftestEscrowCreate(context.Background(), cfg, logger, pbsStorage, paperkey, offline, upload, idBundlePath, directivePath, outputMode)) case "escrow-consume": os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest)) case "identity-consume": @@ -722,7 +724,21 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger) loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner)) - localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, logger, &localTokens) + // Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate — + // the pbsdr manager is constructed further down; the closure reads drConfigured at call time + // (nil until then → preflight reports the tier not applied, which is the honest pre-wire answer). + escrowCeremonyCfg := &localapi.EscrowCeremonyConfig{ + SudoPath: cfg.Privileged.SudoPath, + PBSStorageID: cfg.Escrow.PBSStorageID, + HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "", + DRConfigured: func() bool { + if drConfigured != nil { + return drConfigured() + } + return false + }, + } + localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens) if localTokens != nil { defer localTokens.Close() } @@ -1071,7 +1087,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re // leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the // daemon — the host still reports/reconciles; only the controller channel is unavailable until // fixed. The opened token store is returned via outTokens so the caller can Close it. -func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { +func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { if !cfg.LocalAPI.Enabled() { return nil } @@ -1143,7 +1159,9 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St HostMetrics: collector, HostID: cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op LogRing: logRing, // v0.83.0: GET /debug/logs — the always-DEBUG capture ring - Logger: logger, + // Controller-driven escrow ceremony (v0.88.0): the customer wizard's agent half. + EscrowCeremony: escrowCeremony, + Logger: logger, }) if err != nil { logger.Warn("daemon: local-api disabled (server build)", "err", err) @@ -1798,22 +1816,58 @@ func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.L return 0 } -// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R, -// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque -// blob. R is surfaced to stdout EXACTLY ONCE (never to the logger/journald). With -upload it PUTs -// the opaque blob to the hub. Enrollment-time, root-capable (reads the 0600 key). -func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath string) int { +// escrowCeremonyOpts carries the CLI switches into the shared ceremony core (v0.88.0 extraction — +// the work is identical for both output modes; only the surfacing differs). +type escrowCeremonyOpts struct { + storage string + paperkey bool + offline bool + upload bool + identityBundlePath string + directivePath string +} + +// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the +// --output=json wire object with RecoveryCode left EMPTY (the shells place R themselves, once). +type escrowCeremonyOutcome struct { + R string + Sum escrow.CeremonyOutput + OfflineCopy []byte // text-mode print block (opt-in b) + Paperkey string // text-mode print block (opt-in a) — SECRET-adjacent + Posture escrow.Posture + Storage string // banner data + Identity bool // banner data +} + +// escrowCeremonyErr classifies a ceremony failure so both output modes keep the pre-v0.88.0 exit +// codes and stderr shapes: usage → exit 2; setup/create/upload → exit 1. kind "upload" means R +// was already minted — the text shell still surfaces it before the failure (the customer must +// receive R; the blob simply never reached the hub), exactly the pre-extraction print order. +type escrowCeremonyErr struct { + kind string // "usage" | "setup" | "create" | "upload" + err error +} + +func (e *escrowCeremonyErr) Error() string { return e.err.Error() } + +// escrowCeremony is the shared ceremony core (slice 7 + 10D.1 + fork-4, extracted v0.88.0 for the +// --output=json machine mode): resolve the key, assemble the identity bundle (WG key + staged +// restic password auto-attach), Create (R + self-verified blob), wipe the staged secret, upload +// when asked. It PRINTS NOTHING — the output-mode shells own every byte of stdout/stderr. R is +// returned for the caller to surface exactly once; escrow.Create never logs it and neither do we. +func escrowCeremony(ctx context.Context, cfg config.Config, logger *slog.Logger, opts escrowCeremonyOpts) (escrowCeremonyOutcome, *escrowCeremonyErr) { + var out escrowCeremonyOutcome + storage := opts.storage if storage == "" { storage = cfg.Escrow.PBSStorageID } if storage == "" { - fmt.Fprintln(os.Stderr, "selftest=escrow-create requires -storage (or escrow.pbs_storage_id)") - return 2 + return out, &escrowCeremonyErr{kind: "usage", err: fmt.Errorf("selftest=escrow-create requires -storage (or escrow.pbs_storage_id)")} } + out.Storage = storage keyPath := cfg.Backup.PBSEncKeyPath(storage) if _, err := os.Stat(keyPath); err != nil { - fmt.Fprintf(os.Stderr, "selftest=escrow-create: PBS key for %q not found (%s): %v\n", storage, keyPath, err) - return 1 + return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("PBS key for %q not found (%s): %v", storage, keyPath, err)} } // Slice 10D.1: optionally ALSO wrap the identity bundle under the same R, and carry the non-secret @@ -1821,20 +1875,18 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo // non-secret (pbs repo/ns, expected fingerprint, tunnel id). var identity *escrow.IdentityBundle var directive json.RawMessage - if identityBundlePath != "" { - raw, err := os.ReadFile(identityBundlePath) + if opts.identityBundlePath != "" { + raw, err := os.ReadFile(opts.identityBundlePath) if err != nil { - fmt.Fprintf(os.Stderr, "selftest=escrow-create: reading identity bundle %s: %v\n", identityBundlePath, err) - return 1 + return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("reading identity bundle %s: %v", opts.identityBundlePath, err)} } var b escrow.IdentityBundle if err := json.Unmarshal(raw, &b); err != nil { - fmt.Fprintf(os.Stderr, "selftest=escrow-create: identity bundle is not valid JSON {tunnel_token,pbs_token}: %v\n", err) - return 1 + return out, &escrowCeremonyErr{kind: "setup", err: fmt.Errorf("identity bundle is not valid JSON {tunnel_token,pbs_token}: %v", err)} } identity = &b - if directivePath != "" { - if d, err := os.ReadFile(directivePath); err == nil && json.Valid(d) { + if opts.directivePath != "" { + if d, err := os.ReadFile(opts.directivePath); err == nil && json.Valid(d) { directive = d } } @@ -1850,8 +1902,7 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo } attached, err := escrow.AttachWGKey(probe, wgKeyPath) if err != nil { - fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", err) - return 1 + return out, &escrowCeremonyErr{kind: "setup", err: err} } if attached { identity = probe @@ -1870,8 +1921,7 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo } attached, err := escrow.AttachResticPassword(probe, escrow.StagedResticPasswordPath()) if err != nil { - fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", err) - return 1 + return out, &escrowCeremonyErr{kind: "setup", err: err} } if attached { identity = probe @@ -1882,21 +1932,20 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo logger.Info("escrow: identity bundle: +restic_repo_password") } } + out.Identity = identity != nil - fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, storage, escrow.DefaultPosture, identity != nil) - // NB: nothing about R is logged. The logger never sees R; only stdout does, once. + // NB: nothing about R is logged. The logger never sees R; only the shells surface it, once. logger.Info("escrow: creating zero-knowledge recovery-code escrow", "storage", storage, "key_path", keyPath, "with_identity", identity != nil) R, res, err := escrow.Create(ctx, escrow.CreateOptions{ KeyPath: keyPath, Posture: escrow.Posture(cfg.Escrow.Posture), - WantOfflineCopy: offline, - WantPaperkey: paperkey, + WantOfflineCopy: opts.offline, + WantPaperkey: opts.paperkey, IdentityBundle: identity, }) if err != nil { - fmt.Fprintln(os.Stderr, " [FAIL] escrow create:", err) - return 1 + return out, &escrowCeremonyErr{kind: "create", err: err} } // fork-4: the staged restic password is now sealed inside the R-wrapped blob — wipe the transient // 0600 staging file so it never lingers on disk (field name only; a wipe failure is a loud warn). @@ -1906,37 +1955,149 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo } } - // Surface R EXACTLY ONCE — to stdout, with a write-it-down banner. Never logged/persisted. + out.R = R + out.Posture = res.Posture + out.OfflineCopy = res.OfflineCopy + out.Paperkey = res.Paperkey + out.Sum = escrow.CeremonyOutput{ + Version: escrow.CeremonyOutputVersion, + KeyFingerprint: res.KeyFingerprint, + EntropyBits: res.EntropyBits, + BlobBytes: len(res.Blob), + IdentityBlobBytes: len(res.IdentityBlob), + ResticPwSealed: resticStaged, + } + if opts.upload { + if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil { + // R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets + // the text shell keep the pre-extraction order (R surfaced, THEN the failure). + return out, &escrowCeremonyErr{kind: "upload", err: err} + } + out.Sum.Uploaded = true + } + return out, nil +} + +// printEscrowTextBanner prints the text-mode header line (shared by the success and the +// post-banner failure paths so the pre-extraction output stays byte-identical — including the +// posture: the historical banner always printed the DEFAULT posture, not the result's). +func printEscrowTextBanner(out escrowCeremonyOutcome) { + fmt.Printf("=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v) ===\n", version, out.Storage, escrow.DefaultPosture, out.Identity) +} + +// printEscrowTextRBlock surfaces R EXACTLY ONCE — to stdout, with the write-it-down banner — +// followed by the non-secret blob facts and the opt-in print blocks. Never logged/persisted. +func printEscrowTextRBlock(out *escrowCeremonyOutcome) { fmt.Println() fmt.Println(" ┌──────────────────────────────────────────────────────────────────────┐") fmt.Println(" │ RECOVERY CODE — write it down now. It is shown ONCE and never stored. │") fmt.Println(" │ Without it your offsite backups are unrecoverable, by anyone. │") fmt.Println(" └──────────────────────────────────────────────────────────────────────┘") - fmt.Println(" " + R) + fmt.Println(" " + out.R) fmt.Println() - R = "" // drop our reference promptly + out.R = "" // drop our reference promptly fmt.Printf(" blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · posture %s · ~%.0f bits R\n", - len(res.Blob), res.KeyFingerprint, res.Posture, res.EntropyBits) + out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Posture, out.Sum.EntropyBits) fmt.Println(" self-verify: the blob unwraps back to the key with R (recoverability confirmed)") - if offline && len(res.OfflineCopy) > 0 { + if len(out.OfflineCopy) > 0 { fmt.Println(" --- (b) R-wrapped OFFLINE COPY (print + store; still needs R) ---") - fmt.Println(base64.StdEncoding.EncodeToString(res.OfflineCopy)) + fmt.Println(base64.StdEncoding.EncodeToString(out.OfflineCopy)) } - if paperkey && res.Paperkey != "" { + if out.Paperkey != "" { fmt.Println(" --- (a) RAW PAPERKEY — single-factor, UNREVOCABLE. Store in a safe only. ---") - fmt.Println(res.Paperkey) + fmt.Println(out.Paperkey) } - if len(res.IdentityBlob) > 0 { - fmt.Printf(" identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", len(res.IdentityBlob)) + if out.Sum.IdentityBlobBytes > 0 { + fmt.Printf(" identity escrow: %d bytes (age-wrapped {tunnel,pbs} under the same R) · self-verify OK\n", out.Sum.IdentityBlobBytes) } - if upload { - if err := uploadEscrowBlob(ctx, cfg, res, directive, resticPwSHA256); err != nil { - fmt.Fprintln(os.Stderr, " [FAIL] upload escrow to hub:", err) +} + +// runSelftestEscrowCreate creates the PBS recovery-code escrow (slice 7, doc 03 §8a): generate R, +// wrap the live PBS key under R (zero-knowledge), self-verify recoverability, and emit the opaque +// blob. With -upload it PUTs the opaque blob to the hub. Enrollment-time, root-capable (reads the +// 0600 key). Output modes (v0.88.0): +// - text (default): the historical human output, byte-identical to pre-v0.88.0 — R to stdout +// EXACTLY ONCE inside the write-it-down banner (never to the logger/journald). +// - json: ONE machine-readable JSON object on stdout (escrow.CeremonyOutput — carries R) and +// NOTHING else there; every human/info line goes to stderr; failures exit non-zero with no +// partial JSON. This is the controller-driven ceremony's parse surface (spike §2.3: the text +// banner is positionally brittle). +func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slog.Logger, storage string, paperkey, offline, upload bool, identityBundlePath, directivePath, outputMode string) int { + switch outputMode { + case "", "text", "json": + default: + fmt.Fprintf(os.Stderr, "selftest=escrow-create: unknown -output %q (text|json)\n", outputMode) + return 2 + } + jsonMode := outputMode == "json" + if jsonMode && (offline || paperkey) { + // The opt-in print blocks are PRINT-oriented (base64 blob / paperkey text on stdout) — + // in json mode stdout carries exactly one JSON object, so refuse loudly instead of + // silently dropping what the caller asked for. + fmt.Fprintln(os.Stderr, "selftest=escrow-create: -offline/-paperkey are text-mode only (their output is print-oriented)") + return 2 + } + + out, cerr := escrowCeremony(ctx, cfg, logger, escrowCeremonyOpts{ + storage: storage, paperkey: paperkey, offline: offline, upload: upload, + identityBundlePath: identityBundlePath, directivePath: directivePath, + }) + if cerr != nil { + switch cerr.kind { + case "usage": + fmt.Fprintln(os.Stderr, cerr.err) + return 2 + case "setup": + fmt.Fprintf(os.Stderr, "selftest=escrow-create: %v\n", cerr.err) + return 1 + case "create": + if !jsonMode { + printEscrowTextBanner(out) + } + fmt.Fprintln(os.Stderr, " [FAIL] escrow create:", cerr.err) + return 1 + default: // "upload" — R was minted; text mode still surfaces it (pre-extraction order) + if !jsonMode { + printEscrowTextBanner(out) + printEscrowTextRBlock(&out) + } + out.R = "" + fmt.Fprintln(os.Stderr, " [FAIL] upload escrow to hub:", cerr.err) return 1 } + } + + if jsonMode { + // Human/info lines → stderr (the job runner's diagnostics tail); the ONE JSON object with + // R → stdout. The consumer (localapi ceremony job) extracts R and zeroes its buffers. + fmt.Fprintf(os.Stderr, "=== felhom-agent %s selftest=escrow-create (storage=%s posture=%s identity=%v output=json) ===\n", version, out.Storage, out.Posture, out.Identity) + fmt.Fprintf(os.Stderr, " blob: %d bytes (opaque, R-wrapped) · key fingerprint %s · ~%.0f bits R · self-verify OK\n", + out.Sum.BlobBytes, out.Sum.KeyFingerprint, out.Sum.EntropyBits) + if out.Sum.IdentityBlobBytes > 0 { + fmt.Fprintf(os.Stderr, " identity escrow: %d bytes · restic_pw_sealed=%v\n", out.Sum.IdentityBlobBytes, out.Sum.ResticPwSealed) + } + if out.Sum.Uploaded { + fmt.Fprintln(os.Stderr, " uploaded the opaque blob(s) to the hub (host record); the hub cannot open them") + } + wire := out.Sum + wire.RecoveryCode = out.R + if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil { + fmt.Fprintf(os.Stderr, "selftest=escrow-create: emitting JSON: %v\n", err) + return 1 + } + // Best-effort scrub: drop every R reference promptly. Go's GC may retain stale copies of + // the string backing array — this shrinks the window, it cannot guarantee erasure. + wire.RecoveryCode = "" + out.R = "" + return 0 + } + + printEscrowTextBanner(out) + printEscrowTextRBlock(&out) + if out.Sum.Uploaded { fmt.Println(" uploaded the opaque blob(s) to the hub (host record); the hub cannot open them") } fmt.Println("=== selftest=escrow-create OK ===") diff --git a/configs/felhom-agent.sudoers b/configs/felhom-agent.sudoers index aa3692c..47bfb3d 100644 --- a/configs/felhom-agent.sudoers +++ b/configs/felhom-agent.sudoers @@ -234,6 +234,16 @@ Cmnd_Alias FELHOM_OOB = \ /usr/sbin/nft add element inet felhom_oob operator_ips *, \ /usr/sbin/nft add element inet felhom_oob ssh_port * +# Escrow ceremony (controller-driven, TASK 2026-07-13; mechanics validated by +# SPIKE-controller-escrow-2026-07-13). ONE fixed argv — sudoers matches the argument vector +# byte-for-byte (spike §2.2): any alteration (value, extra flag, order, config path) is refused. +# --config pinned: env_reset strips FELHOM_AGENT_CONFIG and the pin closes alternate-config +# injection. R rides the subprocess stdout pipe only; sudo logs argv = secrets-free. The argv +# MUST stay byte-identical to escrow.CeremonyArgs() (internal/escrow/ceremony.go) — the +# capability manifest entry + TestManifestCoveredBySudoers lock the three copies together. +Cmnd_Alias FELHOM_ESCROW = \ + /usr/local/bin/felhom-agent --config /etc/felhom-agent/agent.json --selftest=escrow-create --upload --output=json + # Node self-heal (CAMPAIGN-3 Part 6, F12-class defense in depth). The ONE fixed unit the appliance # watchdog may (re)start when a boot leaves networking down — the exact command the morning recovery # ran by hand after the F12 host loss. FIXED unit, no glob: this grant alone cannot harm — starting @@ -243,4 +253,4 @@ Cmnd_Alias FELHOM_OOB = \ Cmnd_Alias FELHOM_SELFHEAL = \ /usr/bin/systemctl start networking.service -felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_SELFHEAL +felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_SELFHEAL, FELHOM_ESCROW diff --git a/internal/capability/manifest.go b/internal/capability/manifest.go index fde409d..70d6a47 100644 --- a/internal/capability/manifest.go +++ b/internal/capability/manifest.go @@ -12,7 +12,11 @@ // by a sudoers pattern, catching authoring gaps in CI before they ship. package capability -import "strings" +import ( + "strings" + + "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" +) // Capability is one privileged command the agent depends on. Name is a stable id; Feature is the // human-readable thing that breaks if the grant is missing (used in logs + the operator alert). @@ -158,6 +162,15 @@ var manifest = []Capability{ {"pbsdr-reconcile", "PBS DR storage-entry reconcile (set-only)", "/usr/local/sbin/felhom-pbs-apply", []string{"reconcile", "felhom-pbs", "10.77.0.1", "ns0", "felhom@pbs!ns0", reprFingerprint, "/etc/pve/priv/storage"}, false, ""}, {"pbsdr-grant", "PBS DR storage ACL self-grant", "/usr/local/sbin/felhom-pbs-apply", []string{"grant", "felhom-pbs"}, false, ""}, + // ---- Escrow ceremony (FELHOM_ESCROW, controller-driven, v0.88.0). Critical: the customer + // wizard's whole run path IS this one grant — a dropped line silently breaks every ceremony. + // GatedBy is set EXPLICITLY (the name deliberately says "escrow", not "pbsdr-": the feature is + // the ceremony, but it only exists behind the DR tier — no PBS key, no ceremony). ReprArgs is + // the SHARED argv constant (internal/escrow/ceremony.go) — the exec runner uses the same one, + // so runner ↔ manifest can't drift, and TestManifestCoveredBySudoers locks manifest ↔ sudoers. + // List-mode probe only (`sudo -n -l`), spike-confirmed side-effect-free — never a real ceremony. + {"escrow-ceremony", "customer recovery-code ceremony (controller-driven)", escrow.CeremonyBinary, escrow.CeremonyArgs(), true, GatePBSDR}, + // ---- Agent self-update (FELHOM_SELFUPDATE, D1). NON-critical: self-update is an occasional // operator-driven op, not a steady-state serving path — a degraded grant means "can't // self-update" (fall back to a manual SSH deploy), not a serving outage. The apply repr uses a diff --git a/internal/capability/manifest_test.go b/internal/capability/manifest_test.go index 51e5705..2b00ef2 100644 --- a/internal/capability/manifest_test.go +++ b/internal/capability/manifest_test.go @@ -2,9 +2,12 @@ package capability import ( "os" + "reflect" "regexp" "strings" "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" ) // sudoersPath is the in-repo allowlist, relative to this test file (internal/capability/). @@ -217,6 +220,50 @@ func TestRedProof_DroppedControllerSwapTeeFailsCheck(t *testing.T) { } } +// TestEscrowCeremonyArgvPinned locks the ceremony argv copies together (Scenario G, v0.88.0). +// The exec runner and the manifest entry both consume escrow.CeremonyArgs() (one shared source), +// and TestManifestCoveredBySudoers proves manifest ⊆ sudoers — so pinning the shared source to +// the EXPECTED literal here transitively locks all three: runner == manifest == sudoers. +// Red-proof: mutate one element of the argv in internal/escrow/ceremony.go and THIS test fails +// (and so does the sudoers coverage); a sudoers-side mutation is caught by the existing +// TestRedProof_* machinery. +func TestEscrowCeremonyArgvPinned(t *testing.T) { + wantBinary := "/usr/local/bin/felhom-agent" + wantArgs := []string{"--config", "/etc/felhom-agent/agent.json", "--selftest=escrow-create", "--upload", "--output=json"} + + if escrow.CeremonyBinary != wantBinary { + t.Errorf("escrow.CeremonyBinary = %q, want %q", escrow.CeremonyBinary, wantBinary) + } + if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) { + t.Errorf("escrow.CeremonyArgs() = %q, want %q (the sudoers line + manifest entry must stay byte-identical)", got, wantArgs) + } + + var entry Capability + for _, c := range Manifest() { + if c.Name == "escrow-ceremony" { + entry = c + } + } + if entry.Name == "" { + t.Fatal("manifest missing escrow-ceremony") + } + if entry.Binary != escrow.CeremonyBinary || !reflect.DeepEqual(entry.ReprArgs, escrow.CeremonyArgs()) { + t.Errorf("manifest escrow-ceremony argv diverged from the shared constant: %s %q", entry.Binary, entry.ReprArgs) + } + if !entry.Critical { + t.Error("escrow-ceremony must be Critical (the wizard's whole run path is this one grant)") + } + if entry.GatedBy != GatePBSDR { + t.Errorf("escrow-ceremony GatedBy = %q, want %q (no PBS key → no ceremony; inactive, never red, on a DR-off box)", entry.GatedBy, GatePBSDR) + } + // CeremonyArgs must return a COPY — a caller mutating its slice must not poison the source. + mutated := escrow.CeremonyArgs() + mutated[0] = "--poisoned" + if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) { + t.Error("escrow.CeremonyArgs() shares its backing array — callers can mutate the source") + } +} + // TestWGCapabilityCriticality pins the exact S4 (v0.66.0) Critical set for the FELHOM_WG entries: // the backup path (conf install, unit enable/restart, handshake read) is operator-alert-worthy now // that offsite backups ride the tunnel; the one-time apt install and the deliberate disable diff --git a/internal/capability/probe_test.go b/internal/capability/probe_test.go index e7d89c7..8b77d26 100644 --- a/internal/capability/probe_test.go +++ b/internal/capability/probe_test.go @@ -130,7 +130,9 @@ func TestProbe_GateOffHealthyIsInactive(t *testing.T) { GateActive: func(gate string) bool { return gate != GatePBSDR }, // DR tier OFF } statuses := p.Probe(context.Background()) - for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant"} { + // v0.88.0: escrow-ceremony joins the gate EXPLICITLY (non-pbsdr name, GatedBy literal) — + // the ceremony only exists behind the DR tier (no PBS key, no ceremony). + for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant", "escrow-ceremony"} { s := find(statuses, name) if s.Status != StatusInactive || s.Reason != ReasonInactive { t.Fatalf("%s = %+v, want inactive/%q", name, s, ReasonInactive) @@ -145,8 +147,8 @@ func TestProbe_GateOffHealthyIsInactive(t *testing.T) { if len(degraded) != 0 { t.Fatalf("inactive leaked into degraded: %+v", degraded) } - if ok != total-3 { - t.Fatalf("ok=%d total=%d, want exactly the 3 gated ones non-ok", ok, total) + if ok != total-4 { + t.Fatalf("ok=%d total=%d, want exactly the 4 gated ones non-ok", ok, total) } } @@ -181,11 +183,13 @@ func TestProbe_GateOnOrNilIsOK(t *testing.T) { } } -// The gate rides the pbsdr- name prefix: exactly the pbsdr-* manifest entries are gated, nothing -// else (a regression here would silently un-gate the tier or gate an unrelated capability). +// The gate covers exactly the pbsdr-* entries (name-prefix mechanism) PLUS escrow-ceremony (an +// explicit GatedBy literal — v0.88.0: the ceremony only exists behind the DR tier, but its name +// says what the feature is). Nothing else may be gated (a regression here would silently un-gate +// the tier or gate an unrelated capability). func TestManifest_ExactlyPBSDRGated(t *testing.T) { for _, c := range Manifest() { - wantGated := strings.HasPrefix(c.Name, "pbsdr-") + wantGated := strings.HasPrefix(c.Name, "pbsdr-") || c.Name == "escrow-ceremony" if gated := c.GatedBy == GatePBSDR; gated != wantGated { t.Fatalf("%s: GatedBy=%q, want gated=%v", c.Name, c.GatedBy, wantGated) } diff --git a/internal/escrow/ceremony.go b/internal/escrow/ceremony.go new file mode 100644 index 0000000..a6426ef --- /dev/null +++ b/internal/escrow/ceremony.go @@ -0,0 +1,49 @@ +package escrow + +// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by +// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API +// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers +// matches that vector byte-for-byte (spike §2.2: any alteration — value, extra flag, order, +// config path — is refused), so the argv below is the SINGLE SOURCE OF TRUTH shared by the +// exec (localapi), the capability manifest entry, and (byte-identically) the FELHOM_ESCROW +// sudoers line. Never build it with flag helpers and never normalize `--` to `-` — Go's flag +// package would accept either spelling, sudoers only the literal one. + +// CeremonyBinary is the installed agent binary path the sudoers line pins. +const CeremonyBinary = "/usr/local/bin/felhom-agent" + +// ceremonyArgv is the fixed vector. --config is pinned explicitly: `sudo -n` env_reset strips +// FELHOM_AGENT_CONFIG, and the pin closes env-injection of an alternate config (spike probe (e)). +var ceremonyArgv = []string{ + "--config", "/etc/felhom-agent/agent.json", + "--selftest=escrow-create", + "--upload", + "--output=json", +} + +// CeremonyArgs returns a fresh copy of the fixed argv (callers must not be able to mutate the +// shared source). +func CeremonyArgs() []string { + out := make([]string, len(ceremonyArgv)) + copy(out, ceremonyArgv) + return out +} + +// CeremonyOutput is the --output=json wire contract (version 1): the ONE JSON object json mode +// emits on stdout — nothing else lands there; every human/info line goes to stderr. RecoveryCode +// is the only secret field: the consumer must extract it, hand it to the one-shot claim holder, +// and zero both the parsed struct and the raw stdout buffer. (Best-effort — Go's GC may hold +// stale copies; the discipline still shrinks the exposure window.) +type CeremonyOutput struct { + Version int `json:"version"` + RecoveryCode string `json:"recovery_code"` + KeyFingerprint string `json:"key_fingerprint"` + EntropyBits float64 `json:"entropy_bits"` + BlobBytes int `json:"blob_bytes"` + IdentityBlobBytes int `json:"identity_blob_bytes"` + ResticPwSealed bool `json:"restic_pw_sealed"` + Uploaded bool `json:"uploaded"` +} + +// CeremonyOutputVersion is the current CeremonyOutput.Version value. +const CeremonyOutputVersion = 1 diff --git a/internal/localapi/escrow_ceremony.go b/internal/localapi/escrow_ceremony.go new file mode 100644 index 0000000..c67bab2 --- /dev/null +++ b/internal/localapi/escrow_ceremony.go @@ -0,0 +1,421 @@ +package localapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "strconv" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" +) + +// Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by +// SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via +// `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW +// sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery +// code R IN MEMORY ONLY for a single one-shot claim. +// +// R custody rules (absolute): +// - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must +// be structurally incapable of carrying R). +// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase +// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes"). +// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is +// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest +// answer. No journal, deliberately — persistence is the one property R must never have. +// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for +// failure diagnostics only. + +// EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony). +type EscrowCeremonyConfig struct { + // SudoPath is the sudo binary ("" → "sudo"). + SudoPath string + // PBSStorageID is cfg.Escrow.PBSStorageID ("" = not configured — preflight red). + PBSStorageID string + // HubConfigured: hub url + host id + api key all present (the --upload target). + HubConfigured bool + // DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured, + // late-bound). nil → reported not-applied. + DRConfigured func() bool +} + +// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until +// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned +// spike-shaped output; production is runCeremonySubprocess. +type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error) + +// escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields — +// R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout) +// here would leak it through every snapshot/status copy. +type escrowCeremonyJob struct { + JobID string + Phase string // running | done | failed | unclaimed_void + StartedAt string + UpdatedAt string + KeyFingerprint string + EntropyBits float64 + ResticPwSealed bool + Uploaded bool + Detail string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike) +} + +const ( + escrowPhaseNone = "none" + escrowPhaseRunning = "running" + escrowPhaseDone = "done" + escrowPhaseFailed = "failed" + escrowPhaseVoid = "unclaimed_void" +) + +// escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl. +// upload — 60 s is a ≥25× margin that still absorbs WAN upload latency. +const escrowCeremonyTimeout = 60 * time.Second + +// escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder +// and flips the job to unclaimed_void. +const escrowClaimTTL = 10 * time.Minute + +// runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv, +// stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log). +func runCeremonySubprocess(sudoPath string) ceremonyRunner { + if sudoPath == "" { + sudoPath = "sudo" + } + return func(ctx context.Context) ([]byte, []byte, int, error) { + args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) + cmd := exec.CommandContext(ctx, sudoPath, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + exit := 0 + if cmd.ProcessState != nil { + exit = cmd.ProcessState.ExitCode() + } + if err != nil && exit == 0 { + exit = -1 // start failure / signal — never mistaken for success + } + return stdout.Bytes(), stderr.Bytes(), exit, err + } +} + +// zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made +// before this runs (string conversions, json decoding) — the discipline shrinks the exposure +// window; it cannot guarantee erasure. +func zeroBytes(b []byte) { + for i := range b { + b[i] = 0 + } +} + +// tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 — +// the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED: +// its unclaimed R (if any) is zeroed before the new job takes the slot. +func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool { + s.escrowMu.Lock() + defer s.escrowMu.Unlock() + if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning { + return false + } + s.wipeEscrowRLocked() + cp := *job + s.escrowJob = &cp + return true +} + +// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder. +// R is stored as a byte slice so the claim/TTL paths can zero it in place. +func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) { + s.escrowMu.Lock() + defer s.escrowMu.Unlock() + job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) + cp := *job + s.escrowJob = &cp + if job.Phase == escrowPhaseDone && r != "" { + s.escrowR = []byte(r) + s.escrowRClaimed = false + s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL) + // Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in + // claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop. + jobID := job.JobID + time.AfterFunc(escrowClaimTTL+time.Second, func() { + s.escrowMu.Lock() + defer s.escrowMu.Unlock() + if s.escrowJob != nil && s.escrowJob.JobID == jobID { + s.expireEscrowRLocked() + } + }) + } +} + +// wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu. +func (s *Server) wipeEscrowRLocked() { + zeroBytes(s.escrowR) + s.escrowR = nil +} + +// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job +// flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu. +func (s *Server) expireEscrowRLocked() { + if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed { + return + } + if len(s.escrowR) == 0 { + return + } + s.wipeEscrowRLocked() + s.escrowJob.Phase = escrowPhaseVoid + s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) + s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)", + "job_id", s.escrowJob.JobID) +} + +// checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary +// path; tests jump the clock). Caller holds escrowMu. +func (s *Server) checkEscrowTTLLocked() { + if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed && + len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) { + s.expireEscrowRLocked() + } +} + +// handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv +// sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status +// and claims R once via /escrow/ceremony/claim. +func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) { + if s.escrowCeremony == nil { + writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") + return + } + if r.ContentLength != 0 { + var req struct { + VMID int `json:"vmid"` + } + if !decodeBody(w, r, &req) { + return + } + if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { + return + } + } + job := &escrowCeremonyJob{ + JobID: "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10), + Phase: escrowPhaseRunning, + StartedAt: s.nowFn().UTC().Format(time.RFC3339), + UpdatedAt: s.nowFn().UTC().Format(time.RFC3339), + } + if !s.tryStartEscrowCeremony(job) { + writeErr(w, http.StatusConflict, "an escrow ceremony is already running") + return + } + s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID) + + base := s.baseCtx + if base == nil { + base = context.Background() + } + done := make(chan struct{}) + go func() { + defer close(done) + start := time.Now() + ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout) + defer cancel() + stdout, stderr, exit, runErr := s.ceremonyRun(ctx) + defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out + + if runErr != nil || exit != 0 { + detail := fmt.Sprintf("exit %d", exit) + if runErr != nil { + detail += ": " + runErr.Error() + } + if tail := tailString(stderr, 500); tail != "" { + detail += " | stderr: " + tail + } + job.Phase, job.Detail = escrowPhaseFailed, detail + s.finishEscrowCeremony(job, "") + s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID, + "exit", exit, "duration_ms", time.Since(start).Milliseconds()) + return + } + var out escrow.CeremonyOutput + if err := json.Unmarshal(stdout, &out); err != nil { + job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object" + s.finishEscrowCeremony(job, "") + s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID) + return + } + if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" { + out.RecoveryCode = "" + job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version) + s.finishEscrowCeremony(job, "") + return + } + job.Phase = escrowPhaseDone + job.KeyFingerprint = out.KeyFingerprint + job.EntropyBits = out.EntropyBits + job.ResticPwSealed = out.ResticPwSealed + job.Uploaded = out.Uploaded + s.finishEscrowCeremony(job, out.RecoveryCode) + out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort) + s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)", + "job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded, + "claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds()) + }() + s.escrowDone = done // tests wait on it; production polls the status endpoint + writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": job.JobID, "phase": job.Phase}, "") +} + +// handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary. +// R is structurally absent (it never enters the job struct). Phase "none" after an agent restart +// is the honest crash answer — the controller re-runs; the new ceremony supersedes. +func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) { + if s.escrowCeremony == nil { + writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") + return + } + s.escrowMu.Lock() + s.checkEscrowTTLLocked() + if s.escrowJob == nil { + s.escrowMu.Unlock() + writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone}) + return + } + job := *s.escrowJob + claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0 + expiresIn := 0 + if claimable { + if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 { + expiresIn = int(d.Seconds()) + } + } + claimed := s.escrowRClaimed + s.escrowMu.Unlock() + + writeOK(w, map[string]any{ + "vmid": vmid, "phase": job.Phase, "job_id": job.JobID, + "started_at": job.StartedAt, "updated_at": job.UpdatedAt, + "key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits, + "restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded, + "claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn, + "detail": job.Detail, + }) +} + +// handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim +// returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410. +// The response body is the ONLY place R ever crosses this API; it is never logged. +func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) { + if s.escrowCeremony == nil { + writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") + return + } + s.escrowMu.Lock() + s.checkEscrowTTLLocked() + switch { + case s.escrowJob == nil: + s.escrowMu.Unlock() + writeErr(w, http.StatusNotFound, "no ceremony has run") + return + case s.escrowJob.Phase == escrowPhaseRunning: + s.escrowMu.Unlock() + writeErr(w, http.StatusConflict, "ceremony still running") + return + case s.escrowJob.Phase == escrowPhaseFailed: + s.escrowMu.Unlock() + writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim") + return + case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid: + s.escrowMu.Unlock() + writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes") + return + } + recovery := string(s.escrowR) + s.wipeEscrowRLocked() + s.escrowRClaimed = true + jobID := s.escrowJob.JobID + s.escrowMu.Unlock() + + s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID) + w.Header().Set("Cache-Control", "no-store") + writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery}) + recovery = "" // drop the reference promptly (GC caveat: best-effort) + _ = recovery +} + +// handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item +// is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700 +// root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error +// instead of producing a false-red (or privilege-requiring) preflight row. +func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) { + if s.escrowCeremony == nil { + writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") + return + } + cfg := s.escrowCeremony + type item struct { + ID string `json:"id"` + OK bool `json:"ok"` + Detail string `json:"detail,omitempty"` + } + items := make([]item, 0, 6) + + items = append(items, item{ID: "pbs_storage_id", OK: cfg.PBSStorageID != "", + Detail: map[bool]string{true: cfg.PBSStorageID, false: "escrow.pbs_storage_id not configured"}[cfg.PBSStorageID != ""]}) + + drOK := cfg.DRConfigured != nil && cfg.DRConfigured() + items = append(items, item{ID: "dr_tier", OK: drOK, + Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]}) + + agePath, ageErr := s.escrowLookPath("age") + items = append(items, item{ID: "age_binary", OK: ageErr == nil, + Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]}) + + items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured, + Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]}) + + // Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages + // before every ceremony when offsite is configured; a no-offsite box legitimately has none). + staged := s.statFile(s.escrowStagePath) + items = append(items, item{ID: "staged_secret", OK: staged, + Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]}) + + sudoErr := s.escrowSudoCheck(r.Context()) + sudoDetail := "sudo grant listed (list-mode)" + if sudoErr != nil { + sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)" + } + items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail}) + + allOK := true + for _, it := range items { + if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking + allOK = false + } + } + writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items}) +} + +// checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- ` — a +// sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified +// side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted. +func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error { + if sudoPath == "" { + sudoPath = "sudo" + } + return func(ctx context.Context) error { + args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) + return exec.CommandContext(ctx, sudoPath, args...).Run() + } +} + +// tailString returns the last max chars of a byte buffer as a trimmed string. +func tailString(b []byte, max int) string { + s := string(bytes.TrimSpace(b)) + if len(s) > max { + s = s[len(s)-max:] + } + return s +} diff --git a/internal/localapi/escrow_ceremony_test.go b/internal/localapi/escrow_ceremony_test.go new file mode 100644 index 0000000..f9b12ef --- /dev/null +++ b/internal/localapi/escrow_ceremony_test.go @@ -0,0 +1,360 @@ +package localapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +// Controller-driven escrow ceremony tests (v0.88.0). The runner seam returns canned SPIKE-SHAPED +// JSON — no sudo, no subprocess. The load-bearing assertions are the R custody rules: one-shot +// claim, TTL void, and R structurally absent from every status/snapshot payload. + +const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india" + +// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values). +func cannedCeremonyJSON(r string) string { + return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r) +} + +// newEscrowTestServer builds a server with the ceremony configured and every seam faked. +func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server { + t.Helper() + srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) + srv.escrowCeremony = &EscrowCeremonyConfig{ + PBSStorageID: "felhom-pbs", + HubConfigured: true, + DRConfigured: func() bool { return true }, + } + srv.ceremonyRun = run + srv.escrowSudoCheck = func(context.Context) error { return nil } + srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil } + srv.statFile = func(string) bool { return true } + return srv +} + +// okRunner returns the canned success output and counts invocations. +func okRunner(calls *atomic.Int32) ceremonyRunner { + return func(context.Context) ([]byte, []byte, int, error) { + if calls != nil { + calls.Add(1) + } + return []byte(cannedCeremonyJSON(testR)), []byte("info: ceremony fine\n"), 0, nil + } +} + +// startAndWait POSTs /escrow/ceremony and waits for the detached job to finish. +func startAndWait(t *testing.T, srv *Server, h http.Handler) { + t.Helper() + if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted { + t.Fatalf("start: got %d, want 202 (%s)", w.Code, w.Body.String()) + } + select { + case <-srv.escrowDone: + case <-time.After(5 * time.Second): + t.Fatal("ceremony job did not finish") + } +} + +// Scenario A/D happy path: run → done → claim ONCE (R delivered, no-store) → 410 on re-claim, +// holder zeroed. R never appears in the start or status payloads. +func TestEscrowCeremony_OneShotClaim(t *testing.T) { + var calls atomic.Int32 + srv := newEscrowTestServer(t, okRunner(&calls)) + h := srv.Handler() + + startAndWait(t, srv, h) + if calls.Load() != 1 { + t.Fatalf("runner called %d times, want 1", calls.Load()) + } + + // Status: done + claimable, summary populated, R ABSENT from the whole payload. + st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") + if st.Code != http.StatusOK { + t.Fatalf("status: got %d", st.Code) + } + body := st.Body.String() + if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) { + t.Fatalf("status not done/claimable: %s", body) + } + if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) { + t.Fatalf("summary fields missing: %s", body) + } + assertNoR(t, "status payload", body) + + // First claim → 200 with EXACTLY the canned R + Cache-Control: no-store. + c1 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") + if c1.Code != http.StatusOK { + t.Fatalf("claim 1: got %d (%s)", c1.Code, c1.Body.String()) + } + if cc := c1.Header().Get("Cache-Control"); cc != "no-store" { + t.Fatalf("claim Cache-Control = %q, want no-store", cc) + } + var env struct { + Data struct { + RecoveryCode string `json:"recovery_code"` + } `json:"data"` + } + if err := json.Unmarshal(c1.Body.Bytes(), &env); err != nil || env.Data.RecoveryCode != testR { + t.Fatalf("claim 1 recovery_code = %q, want the canned R", env.Data.RecoveryCode) + } + + // The in-memory holder is gone the moment the claim returns. + srv.escrowMu.Lock() + holder := len(srv.escrowR) + srv.escrowMu.Unlock() + if holder != 0 { + t.Fatal("R holder survived the claim — the wipe-after-claim is missing") + } + + // Second claim → 410 Gone, and no R anywhere in it. + c2 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") + if c2.Code != http.StatusGone { + t.Fatalf("claim 2: got %d, want 410", c2.Code) + } + assertNoR(t, "re-claim payload", c2.Body.String()) + + // Post-claim status: claimed, not claimable, still phase done. + st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "") + if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) { + t.Fatalf("post-claim status wrong: %s", st2.Body.String()) + } +} + +// Scenario D TTL: an unclaimed R past the 10-min TTL is zeroed and the job flips to +// unclaimed_void; the claim answers 410. (The lazy s.now-driven path — the tested primary.) +func TestEscrowCeremony_TTLExpiryVoidsUnclaimedR(t *testing.T) { + srv := newEscrowTestServer(t, okRunner(nil)) + cur := testNow + srv.now = func() time.Time { return cur } + h := srv.Handler() + + startAndWait(t, srv, h) + + cur = cur.Add(escrowClaimTTL + time.Minute) // jump past the TTL + + c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") + if c.Code != http.StatusGone { + t.Fatalf("claim after TTL: got %d, want 410", c.Code) + } + srv.escrowMu.Lock() + holder := len(srv.escrowR) + phase := srv.escrowJob.Phase + srv.escrowMu.Unlock() + if holder != 0 { + t.Fatal("R holder survived the TTL — the expiry wipe is missing") + } + if phase != escrowPhaseVoid { + t.Fatalf("phase after TTL = %q, want %q", phase, escrowPhaseVoid) + } + st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") + if !strings.Contains(st.Body.String(), `"phase":"unclaimed_void"`) { + t.Fatalf("status after TTL: %s", st.Body.String()) + } + assertNoR(t, "void status payload", st.Body.String()) +} + +// Snapshot hygiene (the §10 mutation target): serialize the ENTIRE job struct — the thing every +// snapshot/status copy derives from — and prove the R substring cannot appear in it. Adding R +// (or the raw stdout) to escrowCeremonyJob makes this fail. +func TestEscrowCeremony_JobStructCannotCarryR(t *testing.T) { + srv := newEscrowTestServer(t, okRunner(nil)) + h := srv.Handler() + startAndWait(t, srv, h) + + srv.escrowMu.Lock() + raw, err := json.Marshal(srv.escrowJob) + srv.escrowMu.Unlock() + if err != nil { + t.Fatalf("marshal job: %v", err) + } + assertNoR(t, "serialized job struct", string(raw)) +} + +// Single-flight: a second start while one is RUNNING → 409, and the runner is not re-invoked. +func TestEscrowCeremony_SingleFlight409(t *testing.T) { + release := make(chan struct{}) + var calls atomic.Int32 + srv := newEscrowTestServer(t, func(ctx context.Context) ([]byte, []byte, int, error) { + calls.Add(1) + <-release + return []byte(cannedCeremonyJSON(testR)), nil, 0, nil + }) + h := srv.Handler() + + if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted { + t.Fatalf("start 1: got %d", w.Code) + } + done := srv.escrowDone + if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict { + t.Fatalf("start 2 while running: got %d, want 409", w.Code) + } + if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"running"`) { + t.Fatalf("status while running: %s", st.Body.String()) + } + close(release) + <-done + if calls.Load() != 1 { + t.Fatalf("runner called %d times, want 1 (the 409 must not spawn)", calls.Load()) + } +} + +// A completed-but-unclaimed ceremony is SUPERSEDED by a re-run: the old R is zeroed before the +// new job takes the slot, and the eventual claim yields the NEW code only. +func TestEscrowCeremony_RerunSupersedesUnclaimedR(t *testing.T) { + const newR = "second-run-code-xxxx" + first := true + srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { + r := newR + if first { + r = testR + first = false + } + return []byte(cannedCeremonyJSON(r)), nil, 0, nil + }) + h := srv.Handler() + + startAndWait(t, srv, h) // run 1, R unclaimed + startAndWait(t, srv, h) // run 2 supersedes + + c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") + if c.Code != http.StatusOK { + t.Fatalf("claim: got %d", c.Code) + } + if !strings.Contains(c.Body.String(), newR) { + t.Fatal("claim did not deliver the SECOND run's code") + } + assertNoR(t, "superseding claim payload", c.Body.String()) // the OLD R must be gone +} + +// Failure paths: non-zero exit carries the stderr tail (log-clean per spike) into detail; stdout +// (secret-bearing) NEVER lands there. Unparseable stdout fails without echoing it. A failed job +// answers 409 on claim. +func TestEscrowCeremony_FailurePaths(t *testing.T) { + t.Run("exit nonzero", func(t *testing.T) { + srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { + return []byte("half-a-secret-" + testR), []byte("selftest=escrow-create: PBS key not found"), 1, fmt.Errorf("exit status 1") + }) + h := srv.Handler() + startAndWait(t, srv, h) + st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") + body := st.Body.String() + if !strings.Contains(body, `"phase":"failed"`) || !strings.Contains(body, "PBS key not found") { + t.Fatalf("failed status lacks the stderr tail: %s", body) + } + assertNoR(t, "failed status payload", body) + if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusConflict { + t.Fatalf("claim on failed: got %d, want 409", c.Code) + } + }) + t.Run("unparseable stdout", func(t *testing.T) { + srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { + return []byte("=== human banner leaked " + testR + " ==="), nil, 0, nil + }) + h := srv.Handler() + startAndWait(t, srv, h) + st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") + if !strings.Contains(st.Body.String(), `"phase":"failed"`) { + t.Fatalf("want failed on unparseable stdout: %s", st.Body.String()) + } + assertNoR(t, "unparseable-stdout status", st.Body.String()) + }) + t.Run("wrong version", func(t *testing.T) { + srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { + return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil + }) + h := srv.Handler() + startAndWait(t, srv, h) + if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"failed"`) { + t.Fatalf("want failed on version mismatch: %s", st.Body.String()) + } + }) +} + +// Restart honesty: a fresh process (empty slot) answers "none" / 404 — the controller treats a +// lost job as void and re-runs (crash-safety is in-memory BY DESIGN). +func TestEscrowCeremony_FreshSlotIsNone(t *testing.T) { + srv := newEscrowTestServer(t, okRunner(nil)) + h := srv.Handler() + if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"none"`) { + t.Fatalf("fresh status: %s", st.Body.String()) + } + if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusNotFound { + t.Fatalf("fresh claim: got %d, want 404", c.Code) + } +} + +// Not-configured servers refuse all four routes (the Options.EscrowCeremony nil case). +func TestEscrowCeremony_NotConfigured(t *testing.T) { + srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) + h := srv.Handler() + for _, probe := range []struct{ method, path string }{ + {"GET", "/escrow/preflight"}, {"POST", "/escrow/ceremony"}, + {"GET", "/escrow/ceremony/status"}, {"POST", "/escrow/ceremony/claim"}, + } { + if w := do(t, h, probe.method, probe.path, "A", ""); w.Code != http.StatusServiceUnavailable { + t.Fatalf("%s %s: got %d, want 503", probe.method, probe.path, w.Code) + } + } +} + +// Preflight: the all-green shape, the missing-grant red row, and the staged-secret item being +// INFORMATIONAL (its false never flips the aggregate ok — the controller owns that decision). +func TestEscrowPreflight(t *testing.T) { + srv := newEscrowTestServer(t, okRunner(nil)) + srv.statFile = func(string) bool { return false } // no staged secret + h := srv.Handler() + + w := do(t, h, "GET", "/escrow/preflight", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("preflight: got %d", w.Code) + } + var env struct { + Data struct { + OK bool `json:"ok"` + Items []struct { + ID string `json:"id"` + OK bool `json:"ok"` + } `json:"items"` + } `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil { + t.Fatalf("decode: %v", err) + } + if !env.Data.OK { + t.Fatalf("aggregate ok=false despite only the informational staged_secret being red: %s", w.Body.String()) + } + seen := map[string]bool{} + for _, it := range env.Data.Items { + seen[it.ID] = it.OK + } + for _, id := range []string{"pbs_storage_id", "dr_tier", "age_binary", "hub_upload", "sudo_grant"} { + if !seen[id] { + t.Fatalf("item %s not ok (or missing): %s", id, w.Body.String()) + } + } + if ok, present := seen["staged_secret"]; !present || ok { + t.Fatalf("staged_secret should be present and false: %s", w.Body.String()) + } + + // Missing sudo grant → its row red AND the aggregate red (it is blocking). + srv.escrowSudoCheck = func(context.Context) error { return fmt.Errorf("denied") } + w2 := do(t, h, "GET", "/escrow/preflight", "A", "") + if !strings.Contains(w2.Body.String(), `"ok":false`) || !strings.Contains(w2.Body.String(), "FELHOM_ESCROW") { + t.Fatalf("missing grant should be a red, named row: %s", w2.Body.String()) + } +} + +// assertNoR fails the test if any fragment of the canned R appears in body — the R-handling +// absolute (§9 rule 4) checked at every non-claim surface. +func assertNoR(t *testing.T, where, body string) { + t.Helper() + if strings.Contains(body, testR) || strings.Contains(body, "canary-alpha") { + t.Fatalf("R leaked into %s: %s", where, body) + } +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index 92951e7..e976601 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -9,6 +9,8 @@ import ( "log/slog" "net" "net/http" + "os" + "os/exec" "strconv" "strings" "sync" @@ -99,6 +101,10 @@ type Options struct { // controller-pushed restic repo password (fork-4). "" → escrow.StagedResticPasswordPath() (the // canonical path the escrow-create ceremony reads). Injectable so the stage handler is testable. EscrowStagePath string + // EscrowCeremony wires the controller-driven ceremony endpoints (v0.88.0): POST /escrow/ceremony + // (+/status, /claim one-shot R) and GET /escrow/preflight. OPTIONAL — when nil, those endpoints + // report "not configured". + EscrowCeremony *EscrowCeremonyConfig // ControllerSwap runs guest commands (pct exec) for the agentic controller-update swap (Phase 1). // OPTIONAL — when nil, POST /controller/swap reports "not configured". Satisfied by *GuestBinder. ControllerSwap GuestExecutor @@ -240,6 +246,26 @@ type Server struct { // netReachable is the 2 s TCP endpoint pre-probe (sync fast-fail + classification tiebreak). netReachable func(proto storage.NetworkProtocol, server string) bool + // Controller-driven escrow ceremony (v0.88.0): the single job slot + the ONE-SHOT in-memory R + // holder. R lives ONLY in escrowR (never in the job struct — snapshots must be structurally + // incapable of carrying it) and is zeroed on claim, supersede, or TTL expiry. See + // escrow_ceremony.go for the custody rules. + escrowCeremony *EscrowCeremonyConfig + escrowMu sync.Mutex + escrowJob *escrowCeremonyJob + escrowR []byte + escrowRClaimed bool + escrowRExpiry time.Time + escrowDone <-chan struct{} // closes when the detached job finishes (tests wait on it) + // ceremonyRun executes the fixed-argv sudo self-invocation (tests inject canned JSON). + ceremonyRun ceremonyRunner + // escrowSudoCheck is the preflight's list-mode grant probe (`sudo -n -l -- `). + escrowSudoCheck func(ctx context.Context) error + // escrowLookPath resolves a binary on PATH for preflight (tests inject). + escrowLookPath func(file string) (string, error) + // statFile reports whether a path exists (preflight's staged-secret item; tests inject). + statFile func(path string) bool + baseCtx context.Context // for fire-and-forget backups; set in Run } @@ -300,6 +326,14 @@ func NewServer(o Options) (*Server, error) { s.netMounted = storage.NetworkMountedAt s.netJournal = readUnitJournal s.netReachable = storage.NetworkEndpointReachable + // Controller-driven escrow ceremony (v0.88.0): production seams; tests inject fakes. + s.escrowCeremony = o.EscrowCeremony + if s.escrowCeremony != nil { + s.ceremonyRun = runCeremonySubprocess(s.escrowCeremony.SudoPath) + s.escrowSudoCheck = checkCeremonySudoGrant(s.escrowCeremony.SudoPath) + } + s.escrowLookPath = exec.LookPath + s.statFile = func(path string) bool { _, err := os.Stat(path); return err == nil } if o.ControllerSwap != nil { s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger) } @@ -350,6 +384,13 @@ func (s *Server) Handler() http.Handler { // fork-4 hygiene: wipe the staged secret once escrowed (controller calls this on confirm). Idempotent. mux.HandleFunc("DELETE /escrow/stage-secret", s.withGuest(s.handleWipeStagedEscrowSecret)) + // Controller-driven escrow ceremony (v0.88.0): preflight checklist, the detached root ceremony + // job (fixed-argv sudo self-invocation), its status, and the ONE-SHOT in-memory R claim. + mux.HandleFunc("GET /escrow/preflight", s.withGuest(s.handleEscrowPreflight)) + mux.HandleFunc("POST /escrow/ceremony", s.withGuest(s.handleEscrowCeremonyStart)) + mux.HandleFunc("GET /escrow/ceremony/status", s.withGuest(s.handleEscrowCeremonyStatus)) + mux.HandleFunc("POST /escrow/ceremony/claim", s.withGuest(s.handleEscrowCeremonyClaim)) + // v0.83.0 observability: the agent's always-DEBUG capture ring, for the controller's // Debug page agent tab (same auth/self-scoping wrap as every sibling route). mux.HandleFunc("GET /debug/logs", s.withGuest(s.handleDebugLogs))