v0.88.0: controller-driven escrow ceremony — --output=json machine mode (escrowCeremony extraction, text mode byte-identical), the ONE fixed argv (escrow.CeremonyArgs, shared by exec+manifest+FELHOM_ESCROW sudoers, pin-tested), localapi ceremony job (single-flight, 60s) + one-shot in-memory R claim (10min TTL, unclaimed_void) + preflight; escrow-ceremony capability (Critical, pbs_dr-gated)
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by
|
||||
// SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via
|
||||
// `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW
|
||||
// sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery
|
||||
// code R IN MEMORY ONLY for a single one-shot claim.
|
||||
//
|
||||
// R custody rules (absolute):
|
||||
// - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must
|
||||
// be structurally incapable of carrying R).
|
||||
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
||||
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
|
||||
// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
|
||||
// answer. No journal, deliberately — persistence is the one property R must never have.
|
||||
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
||||
// failure diagnostics only.
|
||||
|
||||
// EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony).
|
||||
type EscrowCeremonyConfig struct {
|
||||
// SudoPath is the sudo binary ("" → "sudo").
|
||||
SudoPath string
|
||||
// PBSStorageID is cfg.Escrow.PBSStorageID ("" = not configured — preflight red).
|
||||
PBSStorageID string
|
||||
// HubConfigured: hub url + host id + api key all present (the --upload target).
|
||||
HubConfigured bool
|
||||
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
||||
// late-bound). nil → reported not-applied.
|
||||
DRConfigured func() bool
|
||||
}
|
||||
|
||||
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
||||
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
||||
// spike-shaped output; production is runCeremonySubprocess.
|
||||
type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error)
|
||||
|
||||
// escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields —
|
||||
// R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout)
|
||||
// here would leak it through every snapshot/status copy.
|
||||
type escrowCeremonyJob struct {
|
||||
JobID string
|
||||
Phase string // running | done | failed | unclaimed_void
|
||||
StartedAt string
|
||||
UpdatedAt string
|
||||
KeyFingerprint string
|
||||
EntropyBits float64
|
||||
ResticPwSealed bool
|
||||
Uploaded bool
|
||||
Detail string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike)
|
||||
}
|
||||
|
||||
const (
|
||||
escrowPhaseNone = "none"
|
||||
escrowPhaseRunning = "running"
|
||||
escrowPhaseDone = "done"
|
||||
escrowPhaseFailed = "failed"
|
||||
escrowPhaseVoid = "unclaimed_void"
|
||||
)
|
||||
|
||||
// escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl.
|
||||
// upload — 60 s is a ≥25× margin that still absorbs WAN upload latency.
|
||||
const escrowCeremonyTimeout = 60 * time.Second
|
||||
|
||||
// escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder
|
||||
// and flips the job to unclaimed_void.
|
||||
const escrowClaimTTL = 10 * time.Minute
|
||||
|
||||
// runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv,
|
||||
// stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log).
|
||||
func runCeremonySubprocess(sudoPath string) ceremonyRunner {
|
||||
if sudoPath == "" {
|
||||
sudoPath = "sudo"
|
||||
}
|
||||
return func(ctx context.Context) ([]byte, []byte, int, error) {
|
||||
args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
|
||||
cmd := exec.CommandContext(ctx, sudoPath, args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
exit := 0
|
||||
if cmd.ProcessState != nil {
|
||||
exit = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
if err != nil && exit == 0 {
|
||||
exit = -1 // start failure / signal — never mistaken for success
|
||||
}
|
||||
return stdout.Bytes(), stderr.Bytes(), exit, err
|
||||
}
|
||||
}
|
||||
|
||||
// zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made
|
||||
// before this runs (string conversions, json decoding) — the discipline shrinks the exposure
|
||||
// window; it cannot guarantee erasure.
|
||||
func zeroBytes(b []byte) {
|
||||
for i := range b {
|
||||
b[i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 —
|
||||
// the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED:
|
||||
// its unclaimed R (if any) is zeroed before the new job takes the slot.
|
||||
func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
||||
return false
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
return true
|
||||
}
|
||||
|
||||
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
||||
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
if job.Phase == escrowPhaseDone && r != "" {
|
||||
s.escrowR = []byte(r)
|
||||
s.escrowRClaimed = false
|
||||
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
||||
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
||||
// claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop.
|
||||
jobID := job.JobID
|
||||
time.AfterFunc(escrowClaimTTL+time.Second, func() {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
if s.escrowJob != nil && s.escrowJob.JobID == jobID {
|
||||
s.expireEscrowRLocked()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu.
|
||||
func (s *Server) wipeEscrowRLocked() {
|
||||
zeroBytes(s.escrowR)
|
||||
s.escrowR = nil
|
||||
}
|
||||
|
||||
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
||||
// flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu.
|
||||
func (s *Server) expireEscrowRLocked() {
|
||||
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
||||
return
|
||||
}
|
||||
if len(s.escrowR) == 0 {
|
||||
return
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowJob.Phase = escrowPhaseVoid
|
||||
s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)",
|
||||
"job_id", s.escrowJob.JobID)
|
||||
}
|
||||
|
||||
// checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary
|
||||
// path; tests jump the clock). Caller holds escrowMu.
|
||||
func (s *Server) checkEscrowTTLLocked() {
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed &&
|
||||
len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) {
|
||||
s.expireEscrowRLocked()
|
||||
}
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv
|
||||
// sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status
|
||||
// and claims R once via /escrow/ceremony/claim.
|
||||
func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
if r.ContentLength != 0 {
|
||||
var req struct {
|
||||
VMID int `json:"vmid"`
|
||||
}
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
job := &escrowCeremonyJob{
|
||||
JobID: "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10),
|
||||
Phase: escrowPhaseRunning,
|
||||
StartedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !s.tryStartEscrowCeremony(job) {
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
||||
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout)
|
||||
defer cancel()
|
||||
stdout, stderr, exit, runErr := s.ceremonyRun(ctx)
|
||||
defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out
|
||||
|
||||
if runErr != nil || exit != 0 {
|
||||
detail := fmt.Sprintf("exit %d", exit)
|
||||
if runErr != nil {
|
||||
detail += ": " + runErr.Error()
|
||||
}
|
||||
if tail := tailString(stderr, 500); tail != "" {
|
||||
detail += " | stderr: " + tail
|
||||
}
|
||||
job.Phase, job.Detail = escrowPhaseFailed, detail
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
||||
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
||||
return
|
||||
}
|
||||
var out escrow.CeremonyOutput
|
||||
if err := json.Unmarshal(stdout, &out); err != nil {
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
||||
return
|
||||
}
|
||||
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
||||
s.finishEscrowCeremony(job, "")
|
||||
return
|
||||
}
|
||||
job.Phase = escrowPhaseDone
|
||||
job.KeyFingerprint = out.KeyFingerprint
|
||||
job.EntropyBits = out.EntropyBits
|
||||
job.ResticPwSealed = out.ResticPwSealed
|
||||
job.Uploaded = out.Uploaded
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode)
|
||||
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
|
||||
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
||||
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
||||
}()
|
||||
s.escrowDone = done // tests wait on it; production polls the status endpoint
|
||||
writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": job.JobID, "phase": job.Phase}, "")
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary.
|
||||
// R is structurally absent (it never enters the job struct). Phase "none" after an agent restart
|
||||
// is the honest crash answer — the controller re-runs; the new ceremony supersedes.
|
||||
func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
s.escrowMu.Lock()
|
||||
s.checkEscrowTTLLocked()
|
||||
if s.escrowJob == nil {
|
||||
s.escrowMu.Unlock()
|
||||
writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone})
|
||||
return
|
||||
}
|
||||
job := *s.escrowJob
|
||||
claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0
|
||||
expiresIn := 0
|
||||
if claimable {
|
||||
if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 {
|
||||
expiresIn = int(d.Seconds())
|
||||
}
|
||||
}
|
||||
claimed := s.escrowRClaimed
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
writeOK(w, map[string]any{
|
||||
"vmid": vmid, "phase": job.Phase, "job_id": job.JobID,
|
||||
"started_at": job.StartedAt, "updated_at": job.UpdatedAt,
|
||||
"key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits,
|
||||
"restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded,
|
||||
"claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn,
|
||||
"detail": job.Detail,
|
||||
})
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim
|
||||
// returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410.
|
||||
// The response body is the ONLY place R ever crosses this API; it is never logged.
|
||||
func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
s.escrowMu.Lock()
|
||||
s.checkEscrowTTLLocked()
|
||||
switch {
|
||||
case s.escrowJob == nil:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusNotFound, "no ceremony has run")
|
||||
return
|
||||
case s.escrowJob.Phase == escrowPhaseRunning:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony still running")
|
||||
return
|
||||
case s.escrowJob.Phase == escrowPhaseFailed:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
||||
return
|
||||
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
||||
return
|
||||
}
|
||||
recovery := string(s.escrowR)
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowRClaimed = true
|
||||
jobID := s.escrowJob.JobID
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery})
|
||||
recovery = "" // drop the reference promptly (GC caveat: best-effort)
|
||||
_ = recovery
|
||||
}
|
||||
|
||||
// handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item
|
||||
// is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700
|
||||
// root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error
|
||||
// instead of producing a false-red (or privilege-requiring) preflight row.
|
||||
func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
cfg := s.escrowCeremony
|
||||
type item struct {
|
||||
ID string `json:"id"`
|
||||
OK bool `json:"ok"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
items := make([]item, 0, 6)
|
||||
|
||||
items = append(items, item{ID: "pbs_storage_id", OK: cfg.PBSStorageID != "",
|
||||
Detail: map[bool]string{true: cfg.PBSStorageID, false: "escrow.pbs_storage_id not configured"}[cfg.PBSStorageID != ""]})
|
||||
|
||||
drOK := cfg.DRConfigured != nil && cfg.DRConfigured()
|
||||
items = append(items, item{ID: "dr_tier", OK: drOK,
|
||||
Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]})
|
||||
|
||||
agePath, ageErr := s.escrowLookPath("age")
|
||||
items = append(items, item{ID: "age_binary", OK: ageErr == nil,
|
||||
Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]})
|
||||
|
||||
items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured,
|
||||
Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]})
|
||||
|
||||
// Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages
|
||||
// before every ceremony when offsite is configured; a no-offsite box legitimately has none).
|
||||
staged := s.statFile(s.escrowStagePath)
|
||||
items = append(items, item{ID: "staged_secret", OK: staged,
|
||||
Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]})
|
||||
|
||||
sudoErr := s.escrowSudoCheck(r.Context())
|
||||
sudoDetail := "sudo grant listed (list-mode)"
|
||||
if sudoErr != nil {
|
||||
sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)"
|
||||
}
|
||||
items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail})
|
||||
|
||||
allOK := true
|
||||
for _, it := range items {
|
||||
if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking
|
||||
allOK = false
|
||||
}
|
||||
}
|
||||
writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items})
|
||||
}
|
||||
|
||||
// checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- <binary> <argv…>` — a
|
||||
// sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified
|
||||
// side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted.
|
||||
func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error {
|
||||
if sudoPath == "" {
|
||||
sudoPath = "sudo"
|
||||
}
|
||||
return func(ctx context.Context) error {
|
||||
args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
|
||||
return exec.CommandContext(ctx, sudoPath, args...).Run()
|
||||
}
|
||||
}
|
||||
|
||||
// tailString returns the last max chars of a byte buffer as a trimmed string.
|
||||
func tailString(b []byte, max int) string {
|
||||
s := string(bytes.TrimSpace(b))
|
||||
if len(s) > max {
|
||||
s = s[len(s)-max:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user