package localapi import ( "bytes" "context" "crypto/rand" "crypto/subtle" "encoding/json" "fmt" "math/big" "net/http" "os/exec" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" ) // Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by // SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via // `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW // sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery // code R IN MEMORY ONLY for a single one-shot claim. // // R custody rules (absolute): // - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must // be structurally incapable of carrying R). // - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase // unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes"). // - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R AND the package it opens, // which is SAFE (R-978: the package was never uploaded, so the hub still holds the previous one // and the customer's previous code still opens it; a re-run starts over). Status reporting "none" // after a restart is the honest answer. No journal, deliberately — persistence is the one // property R must never have. // - R-978: the NEW package reaches the hub only at the claim, and the code is handed out only after // the hub took it. Until then the previous package stays current — so a ceremony that is started // and abandoned (never claimed, superseded, TTL, restart) leaves the customer's written-down code // working. The TTL and a new start both wait while a claim's upload is in flight, so a package is // never stored without its code being handed out by that same claim. // - R-1151: a claim that asks for it (`{"defer_upload": true}`, controller ≥ 0.332.0) hands out the code // and two word positions, and keeps the package until POST /escrow/ceremony/confirm brings those two // words back — the household's typed-back check, now checked HERE, not only in the browser. A code that // was shown and never typed back (the page closed) never replaces the hub's package; the confirm window // (escrowConfirmTTL) ends it. Only the two words are kept for the check, never R. A claim WITHOUT the // flag (a controller that has not updated) keeps the R-978 order: upload at the claim. Pinned by // TestEscrowCeremony_R1151_*. // - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for // failure diagnostics only. // EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony). type EscrowCeremonyConfig struct { // SudoPath is the sudo binary ("" → "sudo"). SudoPath string // PBSStorageID is cfg.Escrow.PBSStorageID at daemon-start ("" = not configured — preflight // red). It is the FALLBACK snapshot; the live value is CurrentPBSStorageID when wired. PBSStorageID string // CurrentPBSStorageID, when set, is called at preflight time to read the LIVE // escrow.pbs_storage_id. The pbsdr bridge SEEDS this key into agent.json on DR convergence // (finishConverged → seedEscrowStorageID); a static snapshot taken at daemon start would then // stay red until a service restart (v0.89.0 live-reload). The wired closure re-reads config // from disk — exactly what the ceremony subprocess itself loads — so the preflight reflects the // real state the bare `--selftest=escrow-create` one-liner will see. nil → the PBSStorageID // snapshot is used (old wiring / tests). CurrentPBSStorageID func() string // HubConfigured: hub url + host id + api key all present (the --upload target). HubConfigured bool // DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured, // late-bound). nil → reported not-applied. DRConfigured func() bool // Upload sends a ceremony's package (CeremonyOutput.PendingUpload, the hub upload body) to this // host's hub record — called by the claim, BEFORE the code is handed out (R-978). nil → a claim of // a pending package is refused (503): a code must never be handed out for a package the hub lacks. Upload func(ctx context.Context, pkg []byte) error } // escrowUploadTimeout bounds the claim-time package upload (one small PUT; the subprocess's whole // ceremony incl. its upload was measured at ~2.4 s). const escrowUploadTimeout = 30 * time.Second // ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until // parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned // spike-shaped output; production is runCeremonySubprocess. type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error) // escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields — // R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout) // here would leak it through every snapshot/status copy. type escrowCeremonyJob struct { JobID string Phase string // running | done | failed | unclaimed_void StartedAt string UpdatedAt string KeyFingerprint string EntropyBits float64 ResticPwSealed bool Uploaded bool Detail string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike) } const ( escrowPhaseNone = "none" escrowPhaseRunning = "running" escrowPhaseDone = "done" escrowPhaseFailed = "failed" escrowPhaseVoid = "unclaimed_void" // escrowPhaseUnconfirmed (R-1151): the code was shown but its words never came back inside the confirm // window — the package was dropped, the hub's previous package (and the previous code) stays current. escrowPhaseUnconfirmed = "unconfirmed_void" ) // escrowConfirmTTL is how long a claimed code waits for its typed-back words (R-1151). Writing ten words down // by hand takes minutes; past this the package is dropped and the household starts again. const escrowConfirmTTL = 30 * time.Minute // escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl. // upload — 60 s is a ≥25× margin that still absorbs WAN upload latency. const escrowCeremonyTimeout = 60 * time.Second // escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder // and flips the job to unclaimed_void. const escrowClaimTTL = 10 * time.Minute // runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv, // stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log). func runCeremonySubprocess(sudoPath string) ceremonyRunner { if sudoPath == "" { sudoPath = "sudo" } return func(ctx context.Context) ([]byte, []byte, int, error) { args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) cmd := exec.CommandContext(ctx, sudoPath, args...) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr err := cmd.Run() exit := 0 if cmd.ProcessState != nil { exit = cmd.ProcessState.ExitCode() } if err != nil && exit == 0 { exit = -1 // start failure / signal — never mistaken for success } return stdout.Bytes(), stderr.Bytes(), exit, err } } // zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made // before this runs (string conversions, json decoding) — the discipline shrinks the exposure // window; it cannot guarantee erasure. func zeroBytes(b []byte) { for i := range b { b[i] = 0 } } // tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 — // the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED: // its unclaimed R (if any) is zeroed before the new job takes the slot. func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool { s.escrowMu.Lock() defer s.escrowMu.Unlock() if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning { return false } if s.escrowClaiming { return false // R-978: a claim is uploading this job's package — its code must reach the claimer } s.wipeEscrowRLocked() cp := *job s.escrowJob = &cp return true } // finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder. // R is stored as a byte slice so the claim/TTL paths can zero it in place. func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string, pending []byte) { s.escrowMu.Lock() defer s.escrowMu.Unlock() job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) cp := *job s.escrowJob = &cp if job.Phase == escrowPhaseDone && r != "" { s.escrowR = []byte(r) s.escrowPending = append([]byte(nil), pending...) s.escrowRClaimed = false s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL) // Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in // claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop. jobID := job.JobID time.AfterFunc(escrowClaimTTL+time.Second, func() { s.escrowMu.Lock() defer s.escrowMu.Unlock() if s.escrowJob != nil && s.escrowJob.JobID == jobID { s.expireEscrowRLocked() } }) } } // wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu. func (s *Server) wipeEscrowRLocked() { zeroBytes(s.escrowR) s.escrowR = nil s.escrowPending = nil // R-978: the package goes with its code — never uploaded once the code is gone s.wipeEscrowConfirmLocked() } // wipeEscrowConfirmLocked drops the R-1151 typed-back state: the two kept words and the wait. Caller holds escrowMu. func (s *Server) wipeEscrowConfirmLocked() { for i := range s.escrowConfirmWords { zeroBytes(s.escrowConfirmWords[i]) s.escrowConfirmWords[i] = nil } s.escrowAwaitConfirm = false } // checkEscrowConfirmTTLLocked ends a typed-back wait that ran past escrowConfirmTTL (R-1151): the package is // dropped (never uploaded) and the job reads unconfirmed_void. Not while a confirm is uploading. Caller holds escrowMu. func (s *Server) checkEscrowConfirmTTLLocked() { if !s.escrowAwaitConfirm || s.escrowClaiming || !s.nowFn().After(s.escrowConfirmExpiry) { return } s.escrowPending = nil s.wipeEscrowConfirmLocked() if s.escrowJob != nil { s.escrowJob.Phase = escrowPhaseUnconfirmed s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) s.logger.Info("local-api: escrow code was not typed back in time — the new package was dropped; the previous one stays current (R-1151)", "job_id", s.escrowJob.JobID) } } // pickConfirmPositions draws two distinct 1-based word positions out of n (n ≥ 2), smaller first. func pickConfirmPositions(n int) ([2]int, error) { a, err := rand.Int(rand.Reader, big.NewInt(int64(n))) if err != nil { return [2]int{}, err } b, err := rand.Int(rand.Reader, big.NewInt(int64(n-1))) if err != nil { return [2]int{}, err } i, j := int(a.Int64()), int(b.Int64()) if j >= i { j++ } if j < i { i, j = j, i } return [2]int{i + 1, j + 1}, nil } // expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job // flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu. func (s *Server) expireEscrowRLocked() { if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed { return } if s.escrowClaiming { return // R-978: the claim is uploading the package; voiding now would store a package nobody gets the code to } if len(s.escrowR) == 0 { return } s.wipeEscrowRLocked() s.escrowJob.Phase = escrowPhaseVoid s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)", "job_id", s.escrowJob.JobID) } // checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary // path; tests jump the clock). Caller holds escrowMu. func (s *Server) checkEscrowTTLLocked() { if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) { s.expireEscrowRLocked() } } // handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv // sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status // and claims R once via /escrow/ceremony/claim. func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } if r.ContentLength != 0 { var req struct { VMID int `json:"vmid"` } if !decodeBody(w, r, &req) { return } if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { return } } job := &escrowCeremonyJob{ JobID: "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10), Phase: escrowPhaseRunning, StartedAt: s.nowFn().UTC().Format(time.RFC3339), UpdatedAt: s.nowFn().UTC().Format(time.RFC3339), } if !s.tryStartEscrowCeremony(job) { writeErr(w, http.StatusConflict, "an escrow ceremony is already running, or its code is being claimed") return } s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID) base := s.baseCtx if base == nil { base = context.Background() } // The reply's fields are read HERE, before the job goroutine may write job.Phase (a data race the race // detector caught, 2026-10-13 — the reply could show the finished phase or a torn value). jobID, phase := job.JobID, job.Phase done := make(chan struct{}) go func() { defer close(done) start := time.Now() ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout) defer cancel() stdout, stderr, exit, runErr := s.ceremonyRun(ctx) defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out if runErr != nil || exit != 0 { detail := fmt.Sprintf("exit %d", exit) if runErr != nil { detail += ": " + runErr.Error() } if tail := tailString(stderr, 500); tail != "" { detail += " | stderr: " + tail } job.Phase, job.Detail = escrowPhaseFailed, detail s.finishEscrowCeremony(job, "", nil) s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID, "exit", exit, "duration_ms", time.Since(start).Milliseconds()) return } var out escrow.CeremonyOutput if err := json.Unmarshal(stdout, &out); err != nil { job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object" s.finishEscrowCeremony(job, "", nil) s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID) return } if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" { out.RecoveryCode = "" job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version) s.finishEscrowCeremony(job, "", nil) return } // R-978: the package must come back un-uploaded. An output that uploaded by itself (or carries no // package) would mean the previous package is already gone, or the new one can never be stored. if out.Uploaded || len(out.PendingUpload) == 0 { out.RecoveryCode = "" job.Phase, job.Detail = escrowPhaseFailed, "ceremony output carries no pending package" s.finishEscrowCeremony(job, "", nil) s.logger.Warn("local-api: escrow ceremony output has no pending package — failed (R-978)", "job_id", job.JobID, "uploaded", out.Uploaded) return } job.Phase = escrowPhaseDone job.KeyFingerprint = out.KeyFingerprint job.EntropyBits = out.EntropyBits job.ResticPwSealed = out.ResticPwSealed job.Uploaded = false // R-978: uploaded by the claim s.finishEscrowCeremony(job, out.RecoveryCode, out.PendingUpload) out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort) s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot); package uploads at the claim, or at the typed-back confirm (R-1151)", "job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded, "claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds()) }() s.escrowDone = done // tests wait on it; production polls the status endpoint writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": jobID, "phase": phase}, "") } // handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary. // R is structurally absent (it never enters the job struct). Phase "none" after an agent restart // is the honest crash answer — the controller re-runs; the new ceremony supersedes. func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } s.escrowMu.Lock() s.checkEscrowTTLLocked() s.checkEscrowConfirmTTLLocked() if s.escrowJob == nil { s.escrowMu.Unlock() writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone}) return } job := *s.escrowJob claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0 expiresIn := 0 if claimable { if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 { expiresIn = int(d.Seconds()) } } claimed := s.escrowRClaimed awaiting := s.escrowAwaitConfirm confirmIn := 0 if awaiting { if d := s.escrowConfirmExpiry.Sub(s.nowFn()); d > 0 { confirmIn = int(d.Seconds()) } } s.escrowMu.Unlock() writeOK(w, map[string]any{ "vmid": vmid, "phase": job.Phase, "job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt, "key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits, "restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded, "claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn, "awaiting_confirm": awaiting, "confirm_expires_in_sec": confirmIn, "detail": job.Detail, }) } // handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim // returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410. // The response body is the ONLY place R ever crosses this API; it is never logged. func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } // R-1151: a controller ≥ 0.332.0 asks for the package to wait for the typed-back words. An older controller // sends `{}` (or nothing) and gets the R-978 order. Unknown fields are refused (decodeBody). var req struct { DeferUpload bool `json:"defer_upload"` } if !decodeBody(w, r, &req) { return } s.escrowMu.Lock() s.checkEscrowTTLLocked() switch { case s.escrowJob == nil: s.escrowMu.Unlock() writeErr(w, http.StatusNotFound, "no ceremony has run") return case s.escrowJob.Phase == escrowPhaseRunning: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "ceremony still running") return case s.escrowJob.Phase == escrowPhaseFailed: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim") return case s.escrowClaiming: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "the recovery code is being claimed — try again in a moment") return case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid: s.escrowMu.Unlock() writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes") return } // R-1151: the deferred order — hand out the code and two word positions; the package waits for // POST /escrow/ceremony/confirm. Needs an uploader (a confirm could never store it otherwise) and a code of ≥ 2 // words (always 10; anything else falls back to the R-978 order rather than to no check). if req.DeferUpload && len(s.escrowPending) > 0 && s.escrowCeremony.Upload != nil { words := strings.Split(string(s.escrowR), escrow.RecoveryCodeSep) if len(words) >= 2 { pos, perr := pickConfirmPositions(len(words)) if perr == nil { s.escrowConfirmPos = pos s.escrowConfirmWords = [2][]byte{[]byte(words[pos[0]-1]), []byte(words[pos[1]-1])} for i := range words { words[i] = "" } s.escrowAwaitConfirm = true s.escrowConfirmExpiry = s.nowFn().Add(escrowConfirmTTL) recovery := string(s.escrowR) zeroBytes(s.escrowR) // R only — the package and the two words stay for the confirm s.escrowR = nil s.escrowRClaimed = true jobID := s.escrowJob.JobID s.escrowMu.Unlock() time.AfterFunc(escrowConfirmTTL+time.Second, func() { s.escrowMu.Lock() defer s.escrowMu.Unlock() if s.escrowJob != nil && s.escrowJob.JobID == jobID { s.checkEscrowConfirmTTLLocked() } }) s.logger.Info("local-api: escrow ceremony R claimed — the package waits for the typed-back words (R-1151)", "vmid", vmid, "job_id", jobID, "confirm_ttl_s", int(escrowConfirmTTL.Seconds())) w.Header().Set("Cache-Control", "no-store") writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery, "confirm_required": true, "verify_positions": []int{pos[0], pos[1]}, "confirm_expires_in_sec": int(escrowConfirmTTL.Seconds())}) recovery = "" _ = recovery return } s.logger.Warn("local-api: escrow claim — could not draw the confirm positions; package uploads at the claim (R-978 order)", "err", perr) } } // R-978: store the NEW package on the hub first; only then hand out the code that opens it. Until this // upload succeeds the hub keeps the previous package, and the customer's previous code keeps working. if len(s.escrowPending) > 0 { if s.escrowCeremony.Upload == nil { s.escrowMu.Unlock() writeErr(w, http.StatusServiceUnavailable, "the recovery package cannot be stored on this host (no upload configured) — the previous code still works") return } pkg := s.escrowPending jobID := s.escrowJob.JobID s.escrowClaiming = true s.escrowMu.Unlock() start := time.Now() base := s.baseCtx if base == nil { base = context.Background() } uctx, cancel := context.WithTimeout(base, escrowUploadTimeout) err := s.escrowCeremony.Upload(uctx, pkg) cancel() s.escrowMu.Lock() s.escrowClaiming = false if err != nil { s.escrowMu.Unlock() s.logger.Warn("local-api: escrow claim — package upload to the hub failed; code kept claimable, previous package stays current (R-978)", "vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds()) writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again") return } s.escrowPending = nil if s.escrowJob != nil && s.escrowJob.JobID == jobID { s.escrowJob.Uploaded = true s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) } s.logger.Info("local-api: escrow claim — package uploaded to the hub; handing out the code (R-978)", "vmid", vmid, "job_id", jobID, "duration_ms", time.Since(start).Milliseconds()) if len(s.escrowR) == 0 { // cannot happen (TTL and supersede wait for the claim) — never hand out nothing s.escrowMu.Unlock() s.logger.Error("local-api: escrow claim — code vanished during the upload (invariant broken)", "job_id", jobID) writeErr(w, http.StatusInternalServerError, "the recovery code was lost during the claim — run a new ceremony") return } } recovery := string(s.escrowR) s.wipeEscrowRLocked() s.escrowRClaimed = true jobID := s.escrowJob.JobID s.escrowMu.Unlock() s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID) w.Header().Set("Cache-Control", "no-store") writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery, "confirm_required": false}) recovery = "" // drop the reference promptly (GC caveat: best-effort) _ = recovery } // handleEscrowCeremonyConfirm is POST /escrow/ceremony/confirm {"words": [w1, w2]} (R-1151): the two words at the // positions the claim named, as the household typed them back. Right → the package goes to the hub now (200); wrong → // 422 and the wait goes on; nothing waiting → 404 (or 410 after the window ran out); a failed upload → 502 and the // wait goes on (a retry may succeed). The words are compared in constant time, case and spaces aside. func (s *Server) handleEscrowCeremonyConfirm(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } var req struct { Words []string `json:"words"` } if !decodeBody(w, r, &req) { return } s.escrowMu.Lock() s.checkEscrowConfirmTTLLocked() switch { case s.escrowClaiming: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "the recovery package is being stored — try again in a moment") return case !s.escrowAwaitConfirm: gone := s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseUnconfirmed s.escrowMu.Unlock() if gone { writeErr(w, http.StatusGone, "the new recovery code was not confirmed in time — the previous code still works; run a new ceremony") return } writeErr(w, http.StatusNotFound, "no recovery code waits for confirmation") return case len(req.Words) != 2: s.escrowMu.Unlock() writeErr(w, http.StatusBadRequest, "two words are needed") return } match := 1 for i := 0; i < 2; i++ { typed := []byte(strings.ToLower(strings.TrimSpace(req.Words[i]))) match &= subtle.ConstantTimeCompare(typed, s.escrowConfirmWords[i]) zeroBytes(typed) } req.Words = nil if match != 1 { s.escrowMu.Unlock() s.logger.Info("local-api: escrow confirm — the typed words do not match; still waiting (R-1151)", "vmid", vmid) writeErr(w, http.StatusUnprocessableEntity, "the words do not match the recovery code") return } if s.escrowCeremony.Upload == nil || len(s.escrowPending) == 0 { // the claim refuses to defer without both s.escrowMu.Unlock() writeErr(w, http.StatusServiceUnavailable, "the recovery package cannot be stored on this host") return } pkg := s.escrowPending jobID := "" if s.escrowJob != nil { jobID = s.escrowJob.JobID } s.escrowClaiming = true s.escrowMu.Unlock() start := time.Now() base := s.baseCtx if base == nil { base = context.Background() } uctx, cancel := context.WithTimeout(base, escrowUploadTimeout) err := s.escrowCeremony.Upload(uctx, pkg) cancel() s.escrowMu.Lock() s.escrowClaiming = false if err != nil { s.escrowMu.Unlock() s.logger.Warn("local-api: escrow confirm — package upload to the hub failed; still waiting, previous package stays current (R-1151)", "vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds()) writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again") return } s.escrowPending = nil s.wipeEscrowConfirmLocked() if s.escrowJob != nil && s.escrowJob.JobID == jobID { s.escrowJob.Uploaded = true s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) } s.escrowMu.Unlock() s.logger.Info("local-api: escrow confirm — words typed back; the new package is on the hub (R-1151)", "vmid", vmid, "job_id", jobID, "duration_ms", time.Since(start).Milliseconds()) writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "uploaded": true}) } // handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item // is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700 // root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error // instead of producing a false-red (or privilege-requiring) preflight row. func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } cfg := s.escrowCeremony type item struct { ID string `json:"id"` OK bool `json:"ok"` Detail string `json:"detail,omitempty"` } items := make([]item, 0, 6) // Live-reload (v0.89.0): prefer the current on-disk value over the daemon-start snapshot, so a // pbsdr convergence that just seeded escrow.pbs_storage_id flips this row green with no restart. storageID := cfg.PBSStorageID if cfg.CurrentPBSStorageID != nil { storageID = cfg.CurrentPBSStorageID() } items = append(items, item{ID: "pbs_storage_id", OK: storageID != "", Detail: map[bool]string{true: storageID, false: "escrow.pbs_storage_id not configured"}[storageID != ""]}) drOK := cfg.DRConfigured != nil && cfg.DRConfigured() items = append(items, item{ID: "dr_tier", OK: drOK, Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]}) agePath, ageErr := s.escrowLookPath("age") items = append(items, item{ID: "age_binary", OK: ageErr == nil, Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]}) items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured, Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]}) // Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages // before every ceremony when offsite is configured; a no-offsite box legitimately has none). staged := s.statFile(s.escrowStagePath) items = append(items, item{ID: "staged_secret", OK: staged, Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]}) sudoErr := s.escrowSudoCheck(r.Context()) sudoDetail := "sudo grant listed (list-mode)" if sudoErr != nil { sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)" } items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail}) allOK := true for _, it := range items { if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking allOK = false } } writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items}) } // checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- ` — a // sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified // side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted. func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error { if sudoPath == "" { sudoPath = "sudo" } return func(ctx context.Context) error { args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) return exec.CommandContext(ctx, sudoPath, args...).Run() } } // tailString returns the last max chars of a byte buffer as a trimmed string. func tailString(b []byte, max int) string { s := string(bytes.TrimSpace(b)) if len(s) > max { s = s[len(s)-max:] } return s }