package localapi import ( "bytes" "context" "encoding/json" "fmt" "net/http" "os/exec" "strconv" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" ) // Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by // SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via // `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW // sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery // code R IN MEMORY ONLY for a single one-shot claim. // // R custody rules (absolute): // - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must // be structurally incapable of carrying R). // - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase // unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes"). // - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is // on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest // answer. No journal, deliberately — persistence is the one property R must never have. // - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for // failure diagnostics only. // EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony). type EscrowCeremonyConfig struct { // SudoPath is the sudo binary ("" → "sudo"). SudoPath string // PBSStorageID is cfg.Escrow.PBSStorageID at daemon-start ("" = not configured — preflight // red). It is the FALLBACK snapshot; the live value is CurrentPBSStorageID when wired. PBSStorageID string // CurrentPBSStorageID, when set, is called at preflight time to read the LIVE // escrow.pbs_storage_id. The pbsdr bridge SEEDS this key into agent.json on DR convergence // (finishConverged → seedEscrowStorageID); a static snapshot taken at daemon start would then // stay red until a service restart (v0.89.0 live-reload). The wired closure re-reads config // from disk — exactly what the ceremony subprocess itself loads — so the preflight reflects the // real state the bare `--selftest=escrow-create` one-liner will see. nil → the PBSStorageID // snapshot is used (old wiring / tests). CurrentPBSStorageID func() string // HubConfigured: hub url + host id + api key all present (the --upload target). HubConfigured bool // DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured, // late-bound). nil → reported not-applied. DRConfigured func() bool } // ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until // parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned // spike-shaped output; production is runCeremonySubprocess. type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error) // escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields — // R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout) // here would leak it through every snapshot/status copy. type escrowCeremonyJob struct { JobID string Phase string // running | done | failed | unclaimed_void StartedAt string UpdatedAt string KeyFingerprint string EntropyBits float64 ResticPwSealed bool Uploaded bool Detail string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike) } const ( escrowPhaseNone = "none" escrowPhaseRunning = "running" escrowPhaseDone = "done" escrowPhaseFailed = "failed" escrowPhaseVoid = "unclaimed_void" ) // escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl. // upload — 60 s is a ≥25× margin that still absorbs WAN upload latency. const escrowCeremonyTimeout = 60 * time.Second // escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder // and flips the job to unclaimed_void. const escrowClaimTTL = 10 * time.Minute // runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv, // stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log). func runCeremonySubprocess(sudoPath string) ceremonyRunner { if sudoPath == "" { sudoPath = "sudo" } return func(ctx context.Context) ([]byte, []byte, int, error) { args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) cmd := exec.CommandContext(ctx, sudoPath, args...) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr err := cmd.Run() exit := 0 if cmd.ProcessState != nil { exit = cmd.ProcessState.ExitCode() } if err != nil && exit == 0 { exit = -1 // start failure / signal — never mistaken for success } return stdout.Bytes(), stderr.Bytes(), exit, err } } // zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made // before this runs (string conversions, json decoding) — the discipline shrinks the exposure // window; it cannot guarantee erasure. func zeroBytes(b []byte) { for i := range b { b[i] = 0 } } // tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 — // the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED: // its unclaimed R (if any) is zeroed before the new job takes the slot. func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool { s.escrowMu.Lock() defer s.escrowMu.Unlock() if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning { return false } s.wipeEscrowRLocked() cp := *job s.escrowJob = &cp return true } // finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder. // R is stored as a byte slice so the claim/TTL paths can zero it in place. func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) { s.escrowMu.Lock() defer s.escrowMu.Unlock() job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) cp := *job s.escrowJob = &cp if job.Phase == escrowPhaseDone && r != "" { s.escrowR = []byte(r) s.escrowRClaimed = false s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL) // Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in // claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop. jobID := job.JobID time.AfterFunc(escrowClaimTTL+time.Second, func() { s.escrowMu.Lock() defer s.escrowMu.Unlock() if s.escrowJob != nil && s.escrowJob.JobID == jobID { s.expireEscrowRLocked() } }) } } // wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu. func (s *Server) wipeEscrowRLocked() { zeroBytes(s.escrowR) s.escrowR = nil } // expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job // flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu. func (s *Server) expireEscrowRLocked() { if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed { return } if len(s.escrowR) == 0 { return } s.wipeEscrowRLocked() s.escrowJob.Phase = escrowPhaseVoid s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)", "job_id", s.escrowJob.JobID) } // checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary // path; tests jump the clock). Caller holds escrowMu. func (s *Server) checkEscrowTTLLocked() { if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) { s.expireEscrowRLocked() } } // handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv // sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status // and claims R once via /escrow/ceremony/claim. func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } if r.ContentLength != 0 { var req struct { VMID int `json:"vmid"` } if !decodeBody(w, r, &req) { return } if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { return } } job := &escrowCeremonyJob{ JobID: "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10), Phase: escrowPhaseRunning, StartedAt: s.nowFn().UTC().Format(time.RFC3339), UpdatedAt: s.nowFn().UTC().Format(time.RFC3339), } if !s.tryStartEscrowCeremony(job) { writeErr(w, http.StatusConflict, "an escrow ceremony is already running") return } s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID) base := s.baseCtx if base == nil { base = context.Background() } done := make(chan struct{}) go func() { defer close(done) start := time.Now() ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout) defer cancel() stdout, stderr, exit, runErr := s.ceremonyRun(ctx) defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out if runErr != nil || exit != 0 { detail := fmt.Sprintf("exit %d", exit) if runErr != nil { detail += ": " + runErr.Error() } if tail := tailString(stderr, 500); tail != "" { detail += " | stderr: " + tail } job.Phase, job.Detail = escrowPhaseFailed, detail s.finishEscrowCeremony(job, "") s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID, "exit", exit, "duration_ms", time.Since(start).Milliseconds()) return } var out escrow.CeremonyOutput if err := json.Unmarshal(stdout, &out); err != nil { job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object" s.finishEscrowCeremony(job, "") s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID) return } if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" { out.RecoveryCode = "" job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version) s.finishEscrowCeremony(job, "") return } job.Phase = escrowPhaseDone job.KeyFingerprint = out.KeyFingerprint job.EntropyBits = out.EntropyBits job.ResticPwSealed = out.ResticPwSealed job.Uploaded = out.Uploaded s.finishEscrowCeremony(job, out.RecoveryCode) out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort) s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)", "job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded, "claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds()) }() s.escrowDone = done // tests wait on it; production polls the status endpoint writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": job.JobID, "phase": job.Phase}, "") } // handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary. // R is structurally absent (it never enters the job struct). Phase "none" after an agent restart // is the honest crash answer — the controller re-runs; the new ceremony supersedes. func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } s.escrowMu.Lock() s.checkEscrowTTLLocked() if s.escrowJob == nil { s.escrowMu.Unlock() writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone}) return } job := *s.escrowJob claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0 expiresIn := 0 if claimable { if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 { expiresIn = int(d.Seconds()) } } claimed := s.escrowRClaimed s.escrowMu.Unlock() writeOK(w, map[string]any{ "vmid": vmid, "phase": job.Phase, "job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt, "key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits, "restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded, "claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn, "detail": job.Detail, }) } // handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim // returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410. // The response body is the ONLY place R ever crosses this API; it is never logged. func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } s.escrowMu.Lock() s.checkEscrowTTLLocked() switch { case s.escrowJob == nil: s.escrowMu.Unlock() writeErr(w, http.StatusNotFound, "no ceremony has run") return case s.escrowJob.Phase == escrowPhaseRunning: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "ceremony still running") return case s.escrowJob.Phase == escrowPhaseFailed: s.escrowMu.Unlock() writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim") return case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid: s.escrowMu.Unlock() writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes") return } recovery := string(s.escrowR) s.wipeEscrowRLocked() s.escrowRClaimed = true jobID := s.escrowJob.JobID s.escrowMu.Unlock() s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID) w.Header().Set("Cache-Control", "no-store") writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery}) recovery = "" // drop the reference promptly (GC caveat: best-effort) _ = recovery } // handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item // is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700 // root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error // instead of producing a false-red (or privilege-requiring) preflight row. func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) { if s.escrowCeremony == nil { writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host") return } cfg := s.escrowCeremony type item struct { ID string `json:"id"` OK bool `json:"ok"` Detail string `json:"detail,omitempty"` } items := make([]item, 0, 6) // Live-reload (v0.89.0): prefer the current on-disk value over the daemon-start snapshot, so a // pbsdr convergence that just seeded escrow.pbs_storage_id flips this row green with no restart. storageID := cfg.PBSStorageID if cfg.CurrentPBSStorageID != nil { storageID = cfg.CurrentPBSStorageID() } items = append(items, item{ID: "pbs_storage_id", OK: storageID != "", Detail: map[bool]string{true: storageID, false: "escrow.pbs_storage_id not configured"}[storageID != ""]}) drOK := cfg.DRConfigured != nil && cfg.DRConfigured() items = append(items, item{ID: "dr_tier", OK: drOK, Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]}) agePath, ageErr := s.escrowLookPath("age") items = append(items, item{ID: "age_binary", OK: ageErr == nil, Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]}) items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured, Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]}) // Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages // before every ceremony when offsite is configured; a no-offsite box legitimately has none). staged := s.statFile(s.escrowStagePath) items = append(items, item{ID: "staged_secret", OK: staged, Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]}) sudoErr := s.escrowSudoCheck(r.Context()) sudoDetail := "sudo grant listed (list-mode)" if sudoErr != nil { sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)" } items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail}) allOK := true for _, it := range items { if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking allOK = false } } writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items}) } // checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- ` — a // sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified // side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted. func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error { if sudoPath == "" { sudoPath = "sudo" } return func(ctx context.Context) error { args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...) return exec.CommandContext(ctx, sudoPath, args...).Run() } } // tailString returns the last max chars of a byte buffer as a trimmed string. func tailString(b []byte, max int) string { s := string(bytes.TrimSpace(b)) if len(s) > max { s = s[len(s)-max:] } return s }