felhom-agent/internal/localapi/escrow_ceremony.go

package localapi

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"strconv"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
)

// Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by
// SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via
// `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW
// sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery
// code R IN MEMORY ONLY for a single one-shot claim.
//
// R custody rules (absolute):
//   - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must
//     be structurally incapable of carrying R).
//   - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
//     unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
//   - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
//     on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
//     answer. No journal, deliberately — persistence is the one property R must never have.
//   - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
//     failure diagnostics only.

// EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony).
type EscrowCeremonyConfig struct {
	// SudoPath is the sudo binary ("" → "sudo").
	SudoPath string
	// PBSStorageID is cfg.Escrow.PBSStorageID at daemon-start ("" = not configured — preflight
	// red). It is the FALLBACK snapshot; the live value is CurrentPBSStorageID when wired.
	PBSStorageID string
	// CurrentPBSStorageID, when set, is called at preflight time to read the LIVE
	// escrow.pbs_storage_id. The pbsdr bridge SEEDS this key into agent.json on DR convergence
	// (finishConverged → seedEscrowStorageID); a static snapshot taken at daemon start would then
	// stay red until a service restart (v0.89.0 live-reload). The wired closure re-reads config
	// from disk — exactly what the ceremony subprocess itself loads — so the preflight reflects the
	// real state the bare `--selftest=escrow-create` one-liner will see. nil → the PBSStorageID
	// snapshot is used (old wiring / tests).
	CurrentPBSStorageID func() string
	// HubConfigured: hub url + host id + api key all present (the --upload target).
	HubConfigured bool
	// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
	// late-bound). nil → reported not-applied.
	DRConfigured func() bool
}

// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
// spike-shaped output; production is runCeremonySubprocess.
type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error)

// escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields —
// R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout)
// here would leak it through every snapshot/status copy.
type escrowCeremonyJob struct {
	JobID          string
	Phase          string // running | done | failed | unclaimed_void
	StartedAt      string
	UpdatedAt      string
	KeyFingerprint string
	EntropyBits    float64
	ResticPwSealed bool
	Uploaded       bool
	Detail         string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike)
}

const (
	escrowPhaseNone    = "none"
	escrowPhaseRunning = "running"
	escrowPhaseDone    = "done"
	escrowPhaseFailed  = "failed"
	escrowPhaseVoid    = "unclaimed_void"
)

// escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl.
// upload — 60 s is a ≥25× margin that still absorbs WAN upload latency.
const escrowCeremonyTimeout = 60 * time.Second

// escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder
// and flips the job to unclaimed_void.
const escrowClaimTTL = 10 * time.Minute

// runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv,
// stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log).
func runCeremonySubprocess(sudoPath string) ceremonyRunner {
	if sudoPath == "" {
		sudoPath = "sudo"
	}
	return func(ctx context.Context) ([]byte, []byte, int, error) {
		args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
		cmd := exec.CommandContext(ctx, sudoPath, args...)
		var stdout, stderr bytes.Buffer
		cmd.Stdout, cmd.Stderr = &stdout, &stderr
		err := cmd.Run()
		exit := 0
		if cmd.ProcessState != nil {
			exit = cmd.ProcessState.ExitCode()
		}
		if err != nil && exit == 0 {
			exit = -1 // start failure / signal — never mistaken for success
		}
		return stdout.Bytes(), stderr.Bytes(), exit, err
	}
}

// zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made
// before this runs (string conversions, json decoding) — the discipline shrinks the exposure
// window; it cannot guarantee erasure.
func zeroBytes(b []byte) {
	for i := range b {
		b[i] = 0
	}
}

// tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 —
// the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED:
// its unclaimed R (if any) is zeroed before the new job takes the slot.
func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
	s.escrowMu.Lock()
	defer s.escrowMu.Unlock()
	if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
		return false
	}
	s.wipeEscrowRLocked()
	cp := *job
	s.escrowJob = &cp
	return true
}

// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
	s.escrowMu.Lock()
	defer s.escrowMu.Unlock()
	job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
	cp := *job
	s.escrowJob = &cp
	if job.Phase == escrowPhaseDone && r != "" {
		s.escrowR = []byte(r)
		s.escrowRClaimed = false
		s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
		// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
		// claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop.
		jobID := job.JobID
		time.AfterFunc(escrowClaimTTL+time.Second, func() {
			s.escrowMu.Lock()
			defer s.escrowMu.Unlock()
			if s.escrowJob != nil && s.escrowJob.JobID == jobID {
				s.expireEscrowRLocked()
			}
		})
	}
}

// wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu.
func (s *Server) wipeEscrowRLocked() {
	zeroBytes(s.escrowR)
	s.escrowR = nil
}

// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
// flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu.
func (s *Server) expireEscrowRLocked() {
	if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
		return
	}
	if len(s.escrowR) == 0 {
		return
	}
	s.wipeEscrowRLocked()
	s.escrowJob.Phase = escrowPhaseVoid
	s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
	s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)",
		"job_id", s.escrowJob.JobID)
}

// checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary
// path; tests jump the clock). Caller holds escrowMu.
func (s *Server) checkEscrowTTLLocked() {
	if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed &&
		len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) {
		s.expireEscrowRLocked()
	}
}

// handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv
// sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status
// and claims R once via /escrow/ceremony/claim.
func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.escrowCeremony == nil {
		writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
		return
	}
	if r.ContentLength != 0 {
		var req struct {
			VMID int `json:"vmid"`
		}
		if !decodeBody(w, r, &req) {
			return
		}
		if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
			return
		}
	}
	job := &escrowCeremonyJob{
		JobID:     "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10),
		Phase:     escrowPhaseRunning,
		StartedAt: s.nowFn().UTC().Format(time.RFC3339),
		UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
	}
	if !s.tryStartEscrowCeremony(job) {
		writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
		return
	}
	s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)

	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	done := make(chan struct{})
	go func() {
		defer close(done)
		start := time.Now()
		ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout)
		defer cancel()
		stdout, stderr, exit, runErr := s.ceremonyRun(ctx)
		defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out

		if runErr != nil || exit != 0 {
			detail := fmt.Sprintf("exit %d", exit)
			if runErr != nil {
				detail += ": " + runErr.Error()
			}
			if tail := tailString(stderr, 500); tail != "" {
				detail += " | stderr: " + tail
			}
			job.Phase, job.Detail = escrowPhaseFailed, detail
			s.finishEscrowCeremony(job, "")
			s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
				"exit", exit, "duration_ms", time.Since(start).Milliseconds())
			return
		}
		var out escrow.CeremonyOutput
		if err := json.Unmarshal(stdout, &out); err != nil {
			job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
			s.finishEscrowCeremony(job, "")
			s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
			return
		}
		if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
			out.RecoveryCode = ""
			job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
			s.finishEscrowCeremony(job, "")
			return
		}
		job.Phase = escrowPhaseDone
		job.KeyFingerprint = out.KeyFingerprint
		job.EntropyBits = out.EntropyBits
		job.ResticPwSealed = out.ResticPwSealed
		job.Uploaded = out.Uploaded
		s.finishEscrowCeremony(job, out.RecoveryCode)
		out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
		s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
			"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
			"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
	}()
	s.escrowDone = done // tests wait on it; production polls the status endpoint
	writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": job.JobID, "phase": job.Phase}, "")
}

// handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary.
// R is structurally absent (it never enters the job struct). Phase "none" after an agent restart
// is the honest crash answer — the controller re-runs; the new ceremony supersedes.
func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.escrowCeremony == nil {
		writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
		return
	}
	s.escrowMu.Lock()
	s.checkEscrowTTLLocked()
	if s.escrowJob == nil {
		s.escrowMu.Unlock()
		writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone})
		return
	}
	job := *s.escrowJob
	claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0
	expiresIn := 0
	if claimable {
		if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 {
			expiresIn = int(d.Seconds())
		}
	}
	claimed := s.escrowRClaimed
	s.escrowMu.Unlock()

	writeOK(w, map[string]any{
		"vmid": vmid, "phase": job.Phase, "job_id": job.JobID,
		"started_at": job.StartedAt, "updated_at": job.UpdatedAt,
		"key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits,
		"restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded,
		"claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn,
		"detail": job.Detail,
	})
}

// handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim
// returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410.
// The response body is the ONLY place R ever crosses this API; it is never logged.
func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.escrowCeremony == nil {
		writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
		return
	}
	s.escrowMu.Lock()
	s.checkEscrowTTLLocked()
	switch {
	case s.escrowJob == nil:
		s.escrowMu.Unlock()
		writeErr(w, http.StatusNotFound, "no ceremony has run")
		return
	case s.escrowJob.Phase == escrowPhaseRunning:
		s.escrowMu.Unlock()
		writeErr(w, http.StatusConflict, "ceremony still running")
		return
	case s.escrowJob.Phase == escrowPhaseFailed:
		s.escrowMu.Unlock()
		writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
		return
	case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
		s.escrowMu.Unlock()
		writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
		return
	}
	recovery := string(s.escrowR)
	s.wipeEscrowRLocked()
	s.escrowRClaimed = true
	jobID := s.escrowJob.JobID
	s.escrowMu.Unlock()

	s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID)
	w.Header().Set("Cache-Control", "no-store")
	writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery})
	recovery = "" // drop the reference promptly (GC caveat: best-effort)
	_ = recovery
}

// handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item
// is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700
// root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error
// instead of producing a false-red (or privilege-requiring) preflight row.
func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.escrowCeremony == nil {
		writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
		return
	}
	cfg := s.escrowCeremony
	type item struct {
		ID     string `json:"id"`
		OK     bool   `json:"ok"`
		Detail string `json:"detail,omitempty"`
	}
	items := make([]item, 0, 6)

	// Live-reload (v0.89.0): prefer the current on-disk value over the daemon-start snapshot, so a
	// pbsdr convergence that just seeded escrow.pbs_storage_id flips this row green with no restart.
	storageID := cfg.PBSStorageID
	if cfg.CurrentPBSStorageID != nil {
		storageID = cfg.CurrentPBSStorageID()
	}
	items = append(items, item{ID: "pbs_storage_id", OK: storageID != "",
		Detail: map[bool]string{true: storageID, false: "escrow.pbs_storage_id not configured"}[storageID != ""]})

	drOK := cfg.DRConfigured != nil && cfg.DRConfigured()
	items = append(items, item{ID: "dr_tier", OK: drOK,
		Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]})

	agePath, ageErr := s.escrowLookPath("age")
	items = append(items, item{ID: "age_binary", OK: ageErr == nil,
		Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]})

	items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured,
		Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]})

	// Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages
	// before every ceremony when offsite is configured; a no-offsite box legitimately has none).
	staged := s.statFile(s.escrowStagePath)
	items = append(items, item{ID: "staged_secret", OK: staged,
		Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]})

	sudoErr := s.escrowSudoCheck(r.Context())
	sudoDetail := "sudo grant listed (list-mode)"
	if sudoErr != nil {
		sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)"
	}
	items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail})

	allOK := true
	for _, it := range items {
		if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking
			allOK = false
		}
	}
	writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items})
}

// checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- <binary> <argv…>` — a
// sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified
// side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted.
func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error {
	if sudoPath == "" {
		sudoPath = "sudo"
	}
	return func(ctx context.Context) error {
		args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
		return exec.CommandContext(ctx, sudoPath, args...).Run()
	}
}

// tailString returns the last max chars of a byte buffer as a trimmed string.
func tailString(b []byte, max int) string {
	s := string(bytes.TrimSpace(b))
	if len(s) > max {
		s = s[len(s)-max:]
	}
	return s
}