v0.88.0: controller-driven escrow ceremony — --output=json machine mode (escrowCeremony extraction, text mode byte-identical), the ONE fixed argv (escrow.CeremonyArgs, shared by exec+manifest+FELHOM_ESCROW sudoers, pin-tested), localapi ceremony job (single-flight, 60s) + one-shot in-memory R claim (10min TTL, unclaimed_void) + preflight; escrow-ceremony capability (Critical, pbs_dr-gated)
This commit is contained in:
@@ -12,7 +12,11 @@
|
||||
// by a sudoers pattern, catching authoring gaps in CI before they ship.
|
||||
package capability
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// Capability is one privileged command the agent depends on. Name is a stable id; Feature is the
|
||||
// human-readable thing that breaks if the grant is missing (used in logs + the operator alert).
|
||||
@@ -158,6 +162,15 @@ var manifest = []Capability{
|
||||
{"pbsdr-reconcile", "PBS DR storage-entry reconcile (set-only)", "/usr/local/sbin/felhom-pbs-apply", []string{"reconcile", "felhom-pbs", "10.77.0.1", "ns0", "felhom@pbs!ns0", reprFingerprint, "/etc/pve/priv/storage"}, false, ""},
|
||||
{"pbsdr-grant", "PBS DR storage ACL self-grant", "/usr/local/sbin/felhom-pbs-apply", []string{"grant", "felhom-pbs"}, false, ""},
|
||||
|
||||
// ---- Escrow ceremony (FELHOM_ESCROW, controller-driven, v0.88.0). Critical: the customer
|
||||
// wizard's whole run path IS this one grant — a dropped line silently breaks every ceremony.
|
||||
// GatedBy is set EXPLICITLY (the name deliberately says "escrow", not "pbsdr-": the feature is
|
||||
// the ceremony, but it only exists behind the DR tier — no PBS key, no ceremony). ReprArgs is
|
||||
// the SHARED argv constant (internal/escrow/ceremony.go) — the exec runner uses the same one,
|
||||
// so runner ↔ manifest can't drift, and TestManifestCoveredBySudoers locks manifest ↔ sudoers.
|
||||
// List-mode probe only (`sudo -n -l`), spike-confirmed side-effect-free — never a real ceremony.
|
||||
{"escrow-ceremony", "customer recovery-code ceremony (controller-driven)", escrow.CeremonyBinary, escrow.CeremonyArgs(), true, GatePBSDR},
|
||||
|
||||
// ---- Agent self-update (FELHOM_SELFUPDATE, D1). NON-critical: self-update is an occasional
|
||||
// operator-driven op, not a steady-state serving path — a degraded grant means "can't
|
||||
// self-update" (fall back to a manual SSH deploy), not a serving outage. The apply repr uses a
|
||||
|
||||
@@ -2,9 +2,12 @@ package capability
|
||||
|
||||
import (
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// sudoersPath is the in-repo allowlist, relative to this test file (internal/capability/).
|
||||
@@ -217,6 +220,50 @@ func TestRedProof_DroppedControllerSwapTeeFailsCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEscrowCeremonyArgvPinned locks the ceremony argv copies together (Scenario G, v0.88.0).
|
||||
// The exec runner and the manifest entry both consume escrow.CeremonyArgs() (one shared source),
|
||||
// and TestManifestCoveredBySudoers proves manifest ⊆ sudoers — so pinning the shared source to
|
||||
// the EXPECTED literal here transitively locks all three: runner == manifest == sudoers.
|
||||
// Red-proof: mutate one element of the argv in internal/escrow/ceremony.go and THIS test fails
|
||||
// (and so does the sudoers coverage); a sudoers-side mutation is caught by the existing
|
||||
// TestRedProof_* machinery.
|
||||
func TestEscrowCeremonyArgvPinned(t *testing.T) {
|
||||
wantBinary := "/usr/local/bin/felhom-agent"
|
||||
wantArgs := []string{"--config", "/etc/felhom-agent/agent.json", "--selftest=escrow-create", "--upload", "--output=json"}
|
||||
|
||||
if escrow.CeremonyBinary != wantBinary {
|
||||
t.Errorf("escrow.CeremonyBinary = %q, want %q", escrow.CeremonyBinary, wantBinary)
|
||||
}
|
||||
if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) {
|
||||
t.Errorf("escrow.CeremonyArgs() = %q, want %q (the sudoers line + manifest entry must stay byte-identical)", got, wantArgs)
|
||||
}
|
||||
|
||||
var entry Capability
|
||||
for _, c := range Manifest() {
|
||||
if c.Name == "escrow-ceremony" {
|
||||
entry = c
|
||||
}
|
||||
}
|
||||
if entry.Name == "" {
|
||||
t.Fatal("manifest missing escrow-ceremony")
|
||||
}
|
||||
if entry.Binary != escrow.CeremonyBinary || !reflect.DeepEqual(entry.ReprArgs, escrow.CeremonyArgs()) {
|
||||
t.Errorf("manifest escrow-ceremony argv diverged from the shared constant: %s %q", entry.Binary, entry.ReprArgs)
|
||||
}
|
||||
if !entry.Critical {
|
||||
t.Error("escrow-ceremony must be Critical (the wizard's whole run path is this one grant)")
|
||||
}
|
||||
if entry.GatedBy != GatePBSDR {
|
||||
t.Errorf("escrow-ceremony GatedBy = %q, want %q (no PBS key → no ceremony; inactive, never red, on a DR-off box)", entry.GatedBy, GatePBSDR)
|
||||
}
|
||||
// CeremonyArgs must return a COPY — a caller mutating its slice must not poison the source.
|
||||
mutated := escrow.CeremonyArgs()
|
||||
mutated[0] = "--poisoned"
|
||||
if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) {
|
||||
t.Error("escrow.CeremonyArgs() shares its backing array — callers can mutate the source")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWGCapabilityCriticality pins the exact S4 (v0.66.0) Critical set for the FELHOM_WG entries:
|
||||
// the backup path (conf install, unit enable/restart, handshake read) is operator-alert-worthy now
|
||||
// that offsite backups ride the tunnel; the one-time apt install and the deliberate disable
|
||||
|
||||
@@ -130,7 +130,9 @@ func TestProbe_GateOffHealthyIsInactive(t *testing.T) {
|
||||
GateActive: func(gate string) bool { return gate != GatePBSDR }, // DR tier OFF
|
||||
}
|
||||
statuses := p.Probe(context.Background())
|
||||
for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant"} {
|
||||
// v0.88.0: escrow-ceremony joins the gate EXPLICITLY (non-pbsdr name, GatedBy literal) —
|
||||
// the ceremony only exists behind the DR tier (no PBS key, no ceremony).
|
||||
for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant", "escrow-ceremony"} {
|
||||
s := find(statuses, name)
|
||||
if s.Status != StatusInactive || s.Reason != ReasonInactive {
|
||||
t.Fatalf("%s = %+v, want inactive/%q", name, s, ReasonInactive)
|
||||
@@ -145,8 +147,8 @@ func TestProbe_GateOffHealthyIsInactive(t *testing.T) {
|
||||
if len(degraded) != 0 {
|
||||
t.Fatalf("inactive leaked into degraded: %+v", degraded)
|
||||
}
|
||||
if ok != total-3 {
|
||||
t.Fatalf("ok=%d total=%d, want exactly the 3 gated ones non-ok", ok, total)
|
||||
if ok != total-4 {
|
||||
t.Fatalf("ok=%d total=%d, want exactly the 4 gated ones non-ok", ok, total)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,11 +183,13 @@ func TestProbe_GateOnOrNilIsOK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The gate rides the pbsdr- name prefix: exactly the pbsdr-* manifest entries are gated, nothing
|
||||
// else (a regression here would silently un-gate the tier or gate an unrelated capability).
|
||||
// The gate covers exactly the pbsdr-* entries (name-prefix mechanism) PLUS escrow-ceremony (an
|
||||
// explicit GatedBy literal — v0.88.0: the ceremony only exists behind the DR tier, but its name
|
||||
// says what the feature is). Nothing else may be gated (a regression here would silently un-gate
|
||||
// the tier or gate an unrelated capability).
|
||||
func TestManifest_ExactlyPBSDRGated(t *testing.T) {
|
||||
for _, c := range Manifest() {
|
||||
wantGated := strings.HasPrefix(c.Name, "pbsdr-")
|
||||
wantGated := strings.HasPrefix(c.Name, "pbsdr-") || c.Name == "escrow-ceremony"
|
||||
if gated := c.GatedBy == GatePBSDR; gated != wantGated {
|
||||
t.Fatalf("%s: GatedBy=%q, want gated=%v", c.Name, c.GatedBy, wantGated)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package escrow
|
||||
|
||||
// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by
|
||||
// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API
|
||||
// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers
|
||||
// matches that vector byte-for-byte (spike §2.2: any alteration — value, extra flag, order,
|
||||
// config path — is refused), so the argv below is the SINGLE SOURCE OF TRUTH shared by the
|
||||
// exec (localapi), the capability manifest entry, and (byte-identically) the FELHOM_ESCROW
|
||||
// sudoers line. Never build it with flag helpers and never normalize `--` to `-` — Go's flag
|
||||
// package would accept either spelling, sudoers only the literal one.
|
||||
|
||||
// CeremonyBinary is the installed agent binary path the sudoers line pins.
|
||||
const CeremonyBinary = "/usr/local/bin/felhom-agent"
|
||||
|
||||
// ceremonyArgv is the fixed vector. --config is pinned explicitly: `sudo -n` env_reset strips
|
||||
// FELHOM_AGENT_CONFIG, and the pin closes env-injection of an alternate config (spike probe (e)).
|
||||
var ceremonyArgv = []string{
|
||||
"--config", "/etc/felhom-agent/agent.json",
|
||||
"--selftest=escrow-create",
|
||||
"--upload",
|
||||
"--output=json",
|
||||
}
|
||||
|
||||
// CeremonyArgs returns a fresh copy of the fixed argv (callers must not be able to mutate the
|
||||
// shared source).
|
||||
func CeremonyArgs() []string {
|
||||
out := make([]string, len(ceremonyArgv))
|
||||
copy(out, ceremonyArgv)
|
||||
return out
|
||||
}
|
||||
|
||||
// CeremonyOutput is the --output=json wire contract (version 1): the ONE JSON object json mode
|
||||
// emits on stdout — nothing else lands there; every human/info line goes to stderr. RecoveryCode
|
||||
// is the only secret field: the consumer must extract it, hand it to the one-shot claim holder,
|
||||
// and zero both the parsed struct and the raw stdout buffer. (Best-effort — Go's GC may hold
|
||||
// stale copies; the discipline still shrinks the exposure window.)
|
||||
type CeremonyOutput struct {
|
||||
Version int `json:"version"`
|
||||
RecoveryCode string `json:"recovery_code"`
|
||||
KeyFingerprint string `json:"key_fingerprint"`
|
||||
EntropyBits float64 `json:"entropy_bits"`
|
||||
BlobBytes int `json:"blob_bytes"`
|
||||
IdentityBlobBytes int `json:"identity_blob_bytes"`
|
||||
ResticPwSealed bool `json:"restic_pw_sealed"`
|
||||
Uploaded bool `json:"uploaded"`
|
||||
}
|
||||
|
||||
// CeremonyOutputVersion is the current CeremonyOutput.Version value.
|
||||
const CeremonyOutputVersion = 1
|
||||
@@ -0,0 +1,421 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// Controller-driven escrow ceremony (v0.88.0, TASK 2026-07-13; every mechanism validated by
|
||||
// SPIKE-controller-escrow-2026-07-13). The daemon re-invokes the agent binary as root via
|
||||
// `sudo -n` with the ONE fixed argv (escrow.CeremonyArgs — byte-identical to the FELHOM_ESCROW
|
||||
// sudoers line), parses the --output=json object off the stdout pipe, and holds the recovery
|
||||
// code R IN MEMORY ONLY for a single one-shot claim.
|
||||
//
|
||||
// R custody rules (absolute):
|
||||
// - R lives in s.escrowR, NEVER inside the job struct (snapshots copy the job; a snapshot must
|
||||
// be structurally incapable of carrying R).
|
||||
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
||||
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
|
||||
// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
|
||||
// answer. No journal, deliberately — persistence is the one property R must never have.
|
||||
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
||||
// failure diagnostics only.
|
||||
|
||||
// EscrowCeremonyConfig wires the ceremony + preflight endpoints (Options.EscrowCeremony).
|
||||
type EscrowCeremonyConfig struct {
|
||||
// SudoPath is the sudo binary ("" → "sudo").
|
||||
SudoPath string
|
||||
// PBSStorageID is cfg.Escrow.PBSStorageID ("" = not configured — preflight red).
|
||||
PBSStorageID string
|
||||
// HubConfigured: hub url + host id + api key all present (the --upload target).
|
||||
HubConfigured bool
|
||||
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
||||
// late-bound). nil → reported not-applied.
|
||||
DRConfigured func() bool
|
||||
}
|
||||
|
||||
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
||||
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
||||
// spike-shaped output; production is runCeremonySubprocess.
|
||||
type ceremonyRunner func(ctx context.Context) (stdout, stderr []byte, exitCode int, err error)
|
||||
|
||||
// escrowCeremonyJob is the single-slot job record. It carries ONLY non-secret summary fields —
|
||||
// R is held separately in Server.escrowR (see the custody rules above); adding R (or raw stdout)
|
||||
// here would leak it through every snapshot/status copy.
|
||||
type escrowCeremonyJob struct {
|
||||
JobID string
|
||||
Phase string // running | done | failed | unclaimed_void
|
||||
StartedAt string
|
||||
UpdatedAt string
|
||||
KeyFingerprint string
|
||||
EntropyBits float64
|
||||
ResticPwSealed bool
|
||||
Uploaded bool
|
||||
Detail string // failure detail: exit code + stderr tail (≤500 chars; log-clean per spike)
|
||||
}
|
||||
|
||||
const (
|
||||
escrowPhaseNone = "none"
|
||||
escrowPhaseRunning = "running"
|
||||
escrowPhaseDone = "done"
|
||||
escrowPhaseFailed = "failed"
|
||||
escrowPhaseVoid = "unclaimed_void"
|
||||
)
|
||||
|
||||
// escrowCeremonyTimeout bounds the whole subprocess run. Spike-measured ceremony ≈ 2.4 s incl.
|
||||
// upload — 60 s is a ≥25× margin that still absorbs WAN upload latency.
|
||||
const escrowCeremonyTimeout = 60 * time.Second
|
||||
|
||||
// escrowClaimTTL is how long a completed ceremony's R stays claimable. Expiry zeroes the holder
|
||||
// and flips the job to unclaimed_void.
|
||||
const escrowClaimTTL = 10 * time.Minute
|
||||
|
||||
// runCeremonySubprocess is the production ceremonyRunner: `sudo -n` + the shared fixed argv,
|
||||
// stdout and stderr captured SEPARATELY (stdout carries R — never merge, never log).
|
||||
func runCeremonySubprocess(sudoPath string) ceremonyRunner {
|
||||
if sudoPath == "" {
|
||||
sudoPath = "sudo"
|
||||
}
|
||||
return func(ctx context.Context) ([]byte, []byte, int, error) {
|
||||
args := append([]string{"-n", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
|
||||
cmd := exec.CommandContext(ctx, sudoPath, args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
exit := 0
|
||||
if cmd.ProcessState != nil {
|
||||
exit = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
if err != nil && exit == 0 {
|
||||
exit = -1 // start failure / signal — never mistaken for success
|
||||
}
|
||||
return stdout.Bytes(), stderr.Bytes(), exit, err
|
||||
}
|
||||
}
|
||||
|
||||
// zeroBytes best-effort-scrubs a secret-bearing buffer. Go's GC may retain stale copies made
|
||||
// before this runs (string conversions, json decoding) — the discipline shrinks the exposure
|
||||
// window; it cannot guarantee erasure.
|
||||
func zeroBytes(b []byte) {
|
||||
for i := range b {
|
||||
b[i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// tryStartEscrowCeremony claims the single ceremony slot. false = one is already RUNNING (409 —
|
||||
// the process peaks ~264 MiB, never allow two). A prior done/failed/void job is SUPERSEDED:
|
||||
// its unclaimed R (if any) is zeroed before the new job takes the slot.
|
||||
func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
||||
return false
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
return true
|
||||
}
|
||||
|
||||
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
||||
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
if job.Phase == escrowPhaseDone && r != "" {
|
||||
s.escrowR = []byte(r)
|
||||
s.escrowRClaimed = false
|
||||
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
||||
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
||||
// claim/status (against s.nowFn) is the tested primary; this is the wall-clock backstop.
|
||||
jobID := job.JobID
|
||||
time.AfterFunc(escrowClaimTTL+time.Second, func() {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
if s.escrowJob != nil && s.escrowJob.JobID == jobID {
|
||||
s.expireEscrowRLocked()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// wipeEscrowRLocked zeroes and drops the R holder (claim, supersede, expiry). Caller holds escrowMu.
|
||||
func (s *Server) wipeEscrowRLocked() {
|
||||
zeroBytes(s.escrowR)
|
||||
s.escrowR = nil
|
||||
}
|
||||
|
||||
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
||||
// flips to unclaimed_void. Claimed or already-void jobs are untouched. Caller holds escrowMu.
|
||||
func (s *Server) expireEscrowRLocked() {
|
||||
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
||||
return
|
||||
}
|
||||
if len(s.escrowR) == 0 {
|
||||
return
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowJob.Phase = escrowPhaseVoid
|
||||
s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
s.logger.Info("local-api: escrow ceremony R expired unclaimed — ceremony void (a re-run supersedes)",
|
||||
"job_id", s.escrowJob.JobID)
|
||||
}
|
||||
|
||||
// checkEscrowTTLLocked lazily applies an elapsed TTL before any read (the s.nowFn-driven primary
|
||||
// path; tests jump the clock). Caller holds escrowMu.
|
||||
func (s *Server) checkEscrowTTLLocked() {
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseDone && !s.escrowRClaimed &&
|
||||
len(s.escrowR) > 0 && s.nowFn().After(s.escrowRExpiry) {
|
||||
s.expireEscrowRLocked()
|
||||
}
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyStart is POST /escrow/ceremony: run the root ceremony via the fixed-argv
|
||||
// sudo self-invocation, detached from the request; the controller polls /escrow/ceremony/status
|
||||
// and claims R once via /escrow/ceremony/claim.
|
||||
func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
if r.ContentLength != 0 {
|
||||
var req struct {
|
||||
VMID int `json:"vmid"`
|
||||
}
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
job := &escrowCeremonyJob{
|
||||
JobID: "escrow-" + strconv.FormatInt(s.nowFn().UnixNano(), 10),
|
||||
Phase: escrowPhaseRunning,
|
||||
StartedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !s.tryStartEscrowCeremony(job) {
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
||||
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(base, escrowCeremonyTimeout)
|
||||
defer cancel()
|
||||
stdout, stderr, exit, runErr := s.ceremonyRun(ctx)
|
||||
defer zeroBytes(stdout) // SECRET-BEARING until parsed; scrub on every path out
|
||||
|
||||
if runErr != nil || exit != 0 {
|
||||
detail := fmt.Sprintf("exit %d", exit)
|
||||
if runErr != nil {
|
||||
detail += ": " + runErr.Error()
|
||||
}
|
||||
if tail := tailString(stderr, 500); tail != "" {
|
||||
detail += " | stderr: " + tail
|
||||
}
|
||||
job.Phase, job.Detail = escrowPhaseFailed, detail
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
||||
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
||||
return
|
||||
}
|
||||
var out escrow.CeremonyOutput
|
||||
if err := json.Unmarshal(stdout, &out); err != nil {
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
||||
return
|
||||
}
|
||||
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
||||
s.finishEscrowCeremony(job, "")
|
||||
return
|
||||
}
|
||||
job.Phase = escrowPhaseDone
|
||||
job.KeyFingerprint = out.KeyFingerprint
|
||||
job.EntropyBits = out.EntropyBits
|
||||
job.ResticPwSealed = out.ResticPwSealed
|
||||
job.Uploaded = out.Uploaded
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode)
|
||||
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
|
||||
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
||||
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
||||
}()
|
||||
s.escrowDone = done // tests wait on it; production polls the status endpoint
|
||||
writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "job_id": job.JobID, "phase": job.Phase}, "")
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyStatus is GET /escrow/ceremony/status: phase + the NON-SECRET summary.
|
||||
// R is structurally absent (it never enters the job struct). Phase "none" after an agent restart
|
||||
// is the honest crash answer — the controller re-runs; the new ceremony supersedes.
|
||||
func (s *Server) handleEscrowCeremonyStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
s.escrowMu.Lock()
|
||||
s.checkEscrowTTLLocked()
|
||||
if s.escrowJob == nil {
|
||||
s.escrowMu.Unlock()
|
||||
writeOK(w, map[string]any{"vmid": vmid, "phase": escrowPhaseNone})
|
||||
return
|
||||
}
|
||||
job := *s.escrowJob
|
||||
claimable := job.Phase == escrowPhaseDone && !s.escrowRClaimed && len(s.escrowR) > 0
|
||||
expiresIn := 0
|
||||
if claimable {
|
||||
if d := s.escrowRExpiry.Sub(s.nowFn()); d > 0 {
|
||||
expiresIn = int(d.Seconds())
|
||||
}
|
||||
}
|
||||
claimed := s.escrowRClaimed
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
writeOK(w, map[string]any{
|
||||
"vmid": vmid, "phase": job.Phase, "job_id": job.JobID,
|
||||
"started_at": job.StartedAt, "updated_at": job.UpdatedAt,
|
||||
"key_fingerprint": job.KeyFingerprint, "entropy_bits": job.EntropyBits,
|
||||
"restic_pw_sealed": job.ResticPwSealed, "uploaded": job.Uploaded,
|
||||
"claimable": claimable, "claimed": claimed, "claim_expires_in_sec": expiresIn,
|
||||
"detail": job.Detail,
|
||||
})
|
||||
}
|
||||
|
||||
// handleEscrowCeremonyClaim is POST /escrow/ceremony/claim — the ONE-SHOT R handoff: first claim
|
||||
// returns {recovery_code} and zeroes the holder; any later claim (or one past the TTL) is 410.
|
||||
// The response body is the ONLY place R ever crosses this API; it is never logged.
|
||||
func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
s.escrowMu.Lock()
|
||||
s.checkEscrowTTLLocked()
|
||||
switch {
|
||||
case s.escrowJob == nil:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusNotFound, "no ceremony has run")
|
||||
return
|
||||
case s.escrowJob.Phase == escrowPhaseRunning:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony still running")
|
||||
return
|
||||
case s.escrowJob.Phase == escrowPhaseFailed:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
||||
return
|
||||
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
||||
return
|
||||
}
|
||||
recovery := string(s.escrowR)
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowRClaimed = true
|
||||
jobID := s.escrowJob.JobID
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
s.logger.Info("local-api: escrow ceremony R claimed (one-shot; holder zeroed)", "vmid", vmid, "job_id", jobID)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeOK(w, map[string]any{"vmid": vmid, "job_id": jobID, "recovery_code": recovery})
|
||||
recovery = "" // drop the reference promptly (GC caveat: best-effort)
|
||||
_ = recovery
|
||||
}
|
||||
|
||||
// handleEscrowPreflight is GET /escrow/preflight: the wizard's prerequisite checklist. Each item
|
||||
// is {id, ok, detail}. Deliberately NOT checked: the PBS key file itself — /etc/pve/priv is 0700
|
||||
// root and the daemon cannot stat it; a missing key fails the ceremony fast with a clear error
|
||||
// instead of producing a false-red (or privilege-requiring) preflight row.
|
||||
func (s *Server) handleEscrowPreflight(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.escrowCeremony == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "escrow ceremony not configured on this host")
|
||||
return
|
||||
}
|
||||
cfg := s.escrowCeremony
|
||||
type item struct {
|
||||
ID string `json:"id"`
|
||||
OK bool `json:"ok"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
items := make([]item, 0, 6)
|
||||
|
||||
items = append(items, item{ID: "pbs_storage_id", OK: cfg.PBSStorageID != "",
|
||||
Detail: map[bool]string{true: cfg.PBSStorageID, false: "escrow.pbs_storage_id not configured"}[cfg.PBSStorageID != ""]})
|
||||
|
||||
drOK := cfg.DRConfigured != nil && cfg.DRConfigured()
|
||||
items = append(items, item{ID: "dr_tier", OK: drOK,
|
||||
Detail: map[bool]string{true: "DR tier applied", false: "DR tier not applied on this host"}[drOK]})
|
||||
|
||||
agePath, ageErr := s.escrowLookPath("age")
|
||||
items = append(items, item{ID: "age_binary", OK: ageErr == nil,
|
||||
Detail: map[bool]string{true: agePath, false: "age binary not installed"}[ageErr == nil]})
|
||||
|
||||
items = append(items, item{ID: "hub_upload", OK: cfg.HubConfigured,
|
||||
Detail: map[bool]string{true: "hub upload target configured", false: "hub url/host_id/api_key incomplete"}[cfg.HubConfigured]})
|
||||
|
||||
// Informational: the CONTROLLER decides whether a missing staged secret matters (it re-stages
|
||||
// before every ceremony when offsite is configured; a no-offsite box legitimately has none).
|
||||
staged := s.statFile(s.escrowStagePath)
|
||||
items = append(items, item{ID: "staged_secret", OK: staged,
|
||||
Detail: map[bool]string{true: "staged secret present", false: "no staged secret (informational — the controller re-stages when offsite is configured)"}[staged]})
|
||||
|
||||
sudoErr := s.escrowSudoCheck(r.Context())
|
||||
sudoDetail := "sudo grant listed (list-mode)"
|
||||
if sudoErr != nil {
|
||||
sudoDetail = "sudoers grant missing (is the FELHOM_ESCROW drop-in installed?)"
|
||||
}
|
||||
items = append(items, item{ID: "sudo_grant", OK: sudoErr == nil, Detail: sudoDetail})
|
||||
|
||||
allOK := true
|
||||
for _, it := range items {
|
||||
if it.ID != "staged_secret" && !it.OK { // staged_secret is informational, never blocking
|
||||
allOK = false
|
||||
}
|
||||
}
|
||||
writeOK(w, map[string]any{"vmid": vmid, "ok": allOK, "items": items})
|
||||
}
|
||||
|
||||
// checkCeremonySudoGrant is the production escrowSudoCheck: `sudo -n -l -- <binary> <argv…>` — a
|
||||
// sudo POLICY LIST that never executes (the capability prober's exact method; spike-verified
|
||||
// side-effect-free on the escrow line). exit 0 ⇔ the fixed argv is permitted.
|
||||
func checkCeremonySudoGrant(sudoPath string) func(ctx context.Context) error {
|
||||
if sudoPath == "" {
|
||||
sudoPath = "sudo"
|
||||
}
|
||||
return func(ctx context.Context) error {
|
||||
args := append([]string{"-n", "-l", "--", escrow.CeremonyBinary}, escrow.CeremonyArgs()...)
|
||||
return exec.CommandContext(ctx, sudoPath, args...).Run()
|
||||
}
|
||||
}
|
||||
|
||||
// tailString returns the last max chars of a byte buffer as a trimmed string.
|
||||
func tailString(b []byte, max int) string {
|
||||
s := string(bytes.TrimSpace(b))
|
||||
if len(s) > max {
|
||||
s = s[len(s)-max:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Controller-driven escrow ceremony tests (v0.88.0). The runner seam returns canned SPIKE-SHAPED
|
||||
// JSON — no sudo, no subprocess. The load-bearing assertions are the R custody rules: one-shot
|
||||
// claim, TTL void, and R structurally absent from every status/snapshot payload.
|
||||
|
||||
const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"
|
||||
|
||||
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values).
|
||||
func cannedCeremonyJSON(r string) string {
|
||||
return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r)
|
||||
}
|
||||
|
||||
// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
|
||||
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
|
||||
t.Helper()
|
||||
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
||||
srv.escrowCeremony = &EscrowCeremonyConfig{
|
||||
PBSStorageID: "felhom-pbs",
|
||||
HubConfigured: true,
|
||||
DRConfigured: func() bool { return true },
|
||||
}
|
||||
srv.ceremonyRun = run
|
||||
srv.escrowSudoCheck = func(context.Context) error { return nil }
|
||||
srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
|
||||
srv.statFile = func(string) bool { return true }
|
||||
return srv
|
||||
}
|
||||
|
||||
// okRunner returns the canned success output and counts invocations.
|
||||
func okRunner(calls *atomic.Int32) ceremonyRunner {
|
||||
return func(context.Context) ([]byte, []byte, int, error) {
|
||||
if calls != nil {
|
||||
calls.Add(1)
|
||||
}
|
||||
return []byte(cannedCeremonyJSON(testR)), []byte("info: ceremony fine\n"), 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
// startAndWait POSTs /escrow/ceremony and waits for the detached job to finish.
|
||||
func startAndWait(t *testing.T, srv *Server, h http.Handler) {
|
||||
t.Helper()
|
||||
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
|
||||
t.Fatalf("start: got %d, want 202 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
select {
|
||||
case <-srv.escrowDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("ceremony job did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario A/D happy path: run → done → claim ONCE (R delivered, no-store) → 410 on re-claim,
|
||||
// holder zeroed. R never appears in the start or status payloads.
|
||||
func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := newEscrowTestServer(t, okRunner(&calls))
|
||||
h := srv.Handler()
|
||||
|
||||
startAndWait(t, srv, h)
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("runner called %d times, want 1", calls.Load())
|
||||
}
|
||||
|
||||
// Status: done + claimable, summary populated, R ABSENT from the whole payload.
|
||||
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
if st.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d", st.Code)
|
||||
}
|
||||
body := st.Body.String()
|
||||
if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
|
||||
t.Fatalf("status not done/claimable: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) {
|
||||
t.Fatalf("summary fields missing: %s", body)
|
||||
}
|
||||
assertNoR(t, "status payload", body)
|
||||
|
||||
// First claim → 200 with EXACTLY the canned R + Cache-Control: no-store.
|
||||
c1 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||
if c1.Code != http.StatusOK {
|
||||
t.Fatalf("claim 1: got %d (%s)", c1.Code, c1.Body.String())
|
||||
}
|
||||
if cc := c1.Header().Get("Cache-Control"); cc != "no-store" {
|
||||
t.Fatalf("claim Cache-Control = %q, want no-store", cc)
|
||||
}
|
||||
var env struct {
|
||||
Data struct {
|
||||
RecoveryCode string `json:"recovery_code"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(c1.Body.Bytes(), &env); err != nil || env.Data.RecoveryCode != testR {
|
||||
t.Fatalf("claim 1 recovery_code = %q, want the canned R", env.Data.RecoveryCode)
|
||||
}
|
||||
|
||||
// The in-memory holder is gone the moment the claim returns.
|
||||
srv.escrowMu.Lock()
|
||||
holder := len(srv.escrowR)
|
||||
srv.escrowMu.Unlock()
|
||||
if holder != 0 {
|
||||
t.Fatal("R holder survived the claim — the wipe-after-claim is missing")
|
||||
}
|
||||
|
||||
// Second claim → 410 Gone, and no R anywhere in it.
|
||||
c2 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||
if c2.Code != http.StatusGone {
|
||||
t.Fatalf("claim 2: got %d, want 410", c2.Code)
|
||||
}
|
||||
assertNoR(t, "re-claim payload", c2.Body.String())
|
||||
|
||||
// Post-claim status: claimed, not claimable, still phase done.
|
||||
st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) {
|
||||
t.Fatalf("post-claim status wrong: %s", st2.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario D TTL: an unclaimed R past the 10-min TTL is zeroed and the job flips to
|
||||
// unclaimed_void; the claim answers 410. (The lazy s.now-driven path — the tested primary.)
|
||||
func TestEscrowCeremony_TTLExpiryVoidsUnclaimedR(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, okRunner(nil))
|
||||
cur := testNow
|
||||
srv.now = func() time.Time { return cur }
|
||||
h := srv.Handler()
|
||||
|
||||
startAndWait(t, srv, h)
|
||||
|
||||
cur = cur.Add(escrowClaimTTL + time.Minute) // jump past the TTL
|
||||
|
||||
c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||
if c.Code != http.StatusGone {
|
||||
t.Fatalf("claim after TTL: got %d, want 410", c.Code)
|
||||
}
|
||||
srv.escrowMu.Lock()
|
||||
holder := len(srv.escrowR)
|
||||
phase := srv.escrowJob.Phase
|
||||
srv.escrowMu.Unlock()
|
||||
if holder != 0 {
|
||||
t.Fatal("R holder survived the TTL — the expiry wipe is missing")
|
||||
}
|
||||
if phase != escrowPhaseVoid {
|
||||
t.Fatalf("phase after TTL = %q, want %q", phase, escrowPhaseVoid)
|
||||
}
|
||||
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
if !strings.Contains(st.Body.String(), `"phase":"unclaimed_void"`) {
|
||||
t.Fatalf("status after TTL: %s", st.Body.String())
|
||||
}
|
||||
assertNoR(t, "void status payload", st.Body.String())
|
||||
}
|
||||
|
||||
// Snapshot hygiene (the §10 mutation target): serialize the ENTIRE job struct — the thing every
|
||||
// snapshot/status copy derives from — and prove the R substring cannot appear in it. Adding R
|
||||
// (or the raw stdout) to escrowCeremonyJob makes this fail.
|
||||
func TestEscrowCeremony_JobStructCannotCarryR(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, okRunner(nil))
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
|
||||
srv.escrowMu.Lock()
|
||||
raw, err := json.Marshal(srv.escrowJob)
|
||||
srv.escrowMu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("marshal job: %v", err)
|
||||
}
|
||||
assertNoR(t, "serialized job struct", string(raw))
|
||||
}
|
||||
|
||||
// Single-flight: a second start while one is RUNNING → 409, and the runner is not re-invoked.
|
||||
func TestEscrowCeremony_SingleFlight409(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
srv := newEscrowTestServer(t, func(ctx context.Context) ([]byte, []byte, int, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return []byte(cannedCeremonyJSON(testR)), nil, 0, nil
|
||||
})
|
||||
h := srv.Handler()
|
||||
|
||||
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
|
||||
t.Fatalf("start 1: got %d", w.Code)
|
||||
}
|
||||
done := srv.escrowDone
|
||||
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict {
|
||||
t.Fatalf("start 2 while running: got %d, want 409", w.Code)
|
||||
}
|
||||
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"running"`) {
|
||||
t.Fatalf("status while running: %s", st.Body.String())
|
||||
}
|
||||
close(release)
|
||||
<-done
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("runner called %d times, want 1 (the 409 must not spawn)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// A completed-but-unclaimed ceremony is SUPERSEDED by a re-run: the old R is zeroed before the
|
||||
// new job takes the slot, and the eventual claim yields the NEW code only.
|
||||
func TestEscrowCeremony_RerunSupersedesUnclaimedR(t *testing.T) {
|
||||
const newR = "second-run-code-xxxx"
|
||||
first := true
|
||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
r := newR
|
||||
if first {
|
||||
r = testR
|
||||
first = false
|
||||
}
|
||||
return []byte(cannedCeremonyJSON(r)), nil, 0, nil
|
||||
})
|
||||
h := srv.Handler()
|
||||
|
||||
startAndWait(t, srv, h) // run 1, R unclaimed
|
||||
startAndWait(t, srv, h) // run 2 supersedes
|
||||
|
||||
c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||
if c.Code != http.StatusOK {
|
||||
t.Fatalf("claim: got %d", c.Code)
|
||||
}
|
||||
if !strings.Contains(c.Body.String(), newR) {
|
||||
t.Fatal("claim did not deliver the SECOND run's code")
|
||||
}
|
||||
assertNoR(t, "superseding claim payload", c.Body.String()) // the OLD R must be gone
|
||||
}
|
||||
|
||||
// Failure paths: non-zero exit carries the stderr tail (log-clean per spike) into detail; stdout
|
||||
// (secret-bearing) NEVER lands there. Unparseable stdout fails without echoing it. A failed job
|
||||
// answers 409 on claim.
|
||||
func TestEscrowCeremony_FailurePaths(t *testing.T) {
|
||||
t.Run("exit nonzero", func(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
return []byte("half-a-secret-" + testR), []byte("selftest=escrow-create: PBS key not found"), 1, fmt.Errorf("exit status 1")
|
||||
})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
body := st.Body.String()
|
||||
if !strings.Contains(body, `"phase":"failed"`) || !strings.Contains(body, "PBS key not found") {
|
||||
t.Fatalf("failed status lacks the stderr tail: %s", body)
|
||||
}
|
||||
assertNoR(t, "failed status payload", body)
|
||||
if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusConflict {
|
||||
t.Fatalf("claim on failed: got %d, want 409", c.Code)
|
||||
}
|
||||
})
|
||||
t.Run("unparseable stdout", func(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
return []byte("=== human banner leaked " + testR + " ==="), nil, 0, nil
|
||||
})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
if !strings.Contains(st.Body.String(), `"phase":"failed"`) {
|
||||
t.Fatalf("want failed on unparseable stdout: %s", st.Body.String())
|
||||
}
|
||||
assertNoR(t, "unparseable-stdout status", st.Body.String())
|
||||
})
|
||||
t.Run("wrong version", func(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil
|
||||
})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"failed"`) {
|
||||
t.Fatalf("want failed on version mismatch: %s", st.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Restart honesty: a fresh process (empty slot) answers "none" / 404 — the controller treats a
|
||||
// lost job as void and re-runs (crash-safety is in-memory BY DESIGN).
|
||||
func TestEscrowCeremony_FreshSlotIsNone(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, okRunner(nil))
|
||||
h := srv.Handler()
|
||||
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"none"`) {
|
||||
t.Fatalf("fresh status: %s", st.Body.String())
|
||||
}
|
||||
if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusNotFound {
|
||||
t.Fatalf("fresh claim: got %d, want 404", c.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Not-configured servers refuse all four routes (the Options.EscrowCeremony nil case).
|
||||
func TestEscrowCeremony_NotConfigured(t *testing.T) {
|
||||
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
||||
h := srv.Handler()
|
||||
for _, probe := range []struct{ method, path string }{
|
||||
{"GET", "/escrow/preflight"}, {"POST", "/escrow/ceremony"},
|
||||
{"GET", "/escrow/ceremony/status"}, {"POST", "/escrow/ceremony/claim"},
|
||||
} {
|
||||
if w := do(t, h, probe.method, probe.path, "A", ""); w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("%s %s: got %d, want 503", probe.method, probe.path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Preflight: the all-green shape, the missing-grant red row, and the staged-secret item being
|
||||
// INFORMATIONAL (its false never flips the aggregate ok — the controller owns that decision).
|
||||
func TestEscrowPreflight(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, okRunner(nil))
|
||||
srv.statFile = func(string) bool { return false } // no staged secret
|
||||
h := srv.Handler()
|
||||
|
||||
w := do(t, h, "GET", "/escrow/preflight", "A", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("preflight: got %d", w.Code)
|
||||
}
|
||||
var env struct {
|
||||
Data struct {
|
||||
OK bool `json:"ok"`
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
OK bool `json:"ok"`
|
||||
} `json:"items"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !env.Data.OK {
|
||||
t.Fatalf("aggregate ok=false despite only the informational staged_secret being red: %s", w.Body.String())
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, it := range env.Data.Items {
|
||||
seen[it.ID] = it.OK
|
||||
}
|
||||
for _, id := range []string{"pbs_storage_id", "dr_tier", "age_binary", "hub_upload", "sudo_grant"} {
|
||||
if !seen[id] {
|
||||
t.Fatalf("item %s not ok (or missing): %s", id, w.Body.String())
|
||||
}
|
||||
}
|
||||
if ok, present := seen["staged_secret"]; !present || ok {
|
||||
t.Fatalf("staged_secret should be present and false: %s", w.Body.String())
|
||||
}
|
||||
|
||||
// Missing sudo grant → its row red AND the aggregate red (it is blocking).
|
||||
srv.escrowSudoCheck = func(context.Context) error { return fmt.Errorf("denied") }
|
||||
w2 := do(t, h, "GET", "/escrow/preflight", "A", "")
|
||||
if !strings.Contains(w2.Body.String(), `"ok":false`) || !strings.Contains(w2.Body.String(), "FELHOM_ESCROW") {
|
||||
t.Fatalf("missing grant should be a red, named row: %s", w2.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// assertNoR fails the test if any fragment of the canned R appears in body — the R-handling
|
||||
// absolute (§9 rule 4) checked at every non-claim surface.
|
||||
func assertNoR(t *testing.T, where, body string) {
|
||||
t.Helper()
|
||||
if strings.Contains(body, testR) || strings.Contains(body, "canary-alpha") {
|
||||
t.Fatalf("R leaked into %s: %s", where, body)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -99,6 +101,10 @@ type Options struct {
|
||||
// controller-pushed restic repo password (fork-4). "" → escrow.StagedResticPasswordPath() (the
|
||||
// canonical path the escrow-create ceremony reads). Injectable so the stage handler is testable.
|
||||
EscrowStagePath string
|
||||
// EscrowCeremony wires the controller-driven ceremony endpoints (v0.88.0): POST /escrow/ceremony
|
||||
// (+/status, /claim one-shot R) and GET /escrow/preflight. OPTIONAL — when nil, those endpoints
|
||||
// report "not configured".
|
||||
EscrowCeremony *EscrowCeremonyConfig
|
||||
// ControllerSwap runs guest commands (pct exec) for the agentic controller-update swap (Phase 1).
|
||||
// OPTIONAL — when nil, POST /controller/swap reports "not configured". Satisfied by *GuestBinder.
|
||||
ControllerSwap GuestExecutor
|
||||
@@ -240,6 +246,26 @@ type Server struct {
|
||||
// netReachable is the 2 s TCP endpoint pre-probe (sync fast-fail + classification tiebreak).
|
||||
netReachable func(proto storage.NetworkProtocol, server string) bool
|
||||
|
||||
// Controller-driven escrow ceremony (v0.88.0): the single job slot + the ONE-SHOT in-memory R
|
||||
// holder. R lives ONLY in escrowR (never in the job struct — snapshots must be structurally
|
||||
// incapable of carrying it) and is zeroed on claim, supersede, or TTL expiry. See
|
||||
// escrow_ceremony.go for the custody rules.
|
||||
escrowCeremony *EscrowCeremonyConfig
|
||||
escrowMu sync.Mutex
|
||||
escrowJob *escrowCeremonyJob
|
||||
escrowR []byte
|
||||
escrowRClaimed bool
|
||||
escrowRExpiry time.Time
|
||||
escrowDone <-chan struct{} // closes when the detached job finishes (tests wait on it)
|
||||
// ceremonyRun executes the fixed-argv sudo self-invocation (tests inject canned JSON).
|
||||
ceremonyRun ceremonyRunner
|
||||
// escrowSudoCheck is the preflight's list-mode grant probe (`sudo -n -l -- <argv>`).
|
||||
escrowSudoCheck func(ctx context.Context) error
|
||||
// escrowLookPath resolves a binary on PATH for preflight (tests inject).
|
||||
escrowLookPath func(file string) (string, error)
|
||||
// statFile reports whether a path exists (preflight's staged-secret item; tests inject).
|
||||
statFile func(path string) bool
|
||||
|
||||
baseCtx context.Context // for fire-and-forget backups; set in Run
|
||||
}
|
||||
|
||||
@@ -300,6 +326,14 @@ func NewServer(o Options) (*Server, error) {
|
||||
s.netMounted = storage.NetworkMountedAt
|
||||
s.netJournal = readUnitJournal
|
||||
s.netReachable = storage.NetworkEndpointReachable
|
||||
// Controller-driven escrow ceremony (v0.88.0): production seams; tests inject fakes.
|
||||
s.escrowCeremony = o.EscrowCeremony
|
||||
if s.escrowCeremony != nil {
|
||||
s.ceremonyRun = runCeremonySubprocess(s.escrowCeremony.SudoPath)
|
||||
s.escrowSudoCheck = checkCeremonySudoGrant(s.escrowCeremony.SudoPath)
|
||||
}
|
||||
s.escrowLookPath = exec.LookPath
|
||||
s.statFile = func(path string) bool { _, err := os.Stat(path); return err == nil }
|
||||
if o.ControllerSwap != nil {
|
||||
s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger)
|
||||
}
|
||||
@@ -350,6 +384,13 @@ func (s *Server) Handler() http.Handler {
|
||||
// fork-4 hygiene: wipe the staged secret once escrowed (controller calls this on confirm). Idempotent.
|
||||
mux.HandleFunc("DELETE /escrow/stage-secret", s.withGuest(s.handleWipeStagedEscrowSecret))
|
||||
|
||||
// Controller-driven escrow ceremony (v0.88.0): preflight checklist, the detached root ceremony
|
||||
// job (fixed-argv sudo self-invocation), its status, and the ONE-SHOT in-memory R claim.
|
||||
mux.HandleFunc("GET /escrow/preflight", s.withGuest(s.handleEscrowPreflight))
|
||||
mux.HandleFunc("POST /escrow/ceremony", s.withGuest(s.handleEscrowCeremonyStart))
|
||||
mux.HandleFunc("GET /escrow/ceremony/status", s.withGuest(s.handleEscrowCeremonyStatus))
|
||||
mux.HandleFunc("POST /escrow/ceremony/claim", s.withGuest(s.handleEscrowCeremonyClaim))
|
||||
|
||||
// v0.83.0 observability: the agent's always-DEBUG capture ring, for the controller's
|
||||
// Debug page agent tab (same auth/self-scoping wrap as every sibling route).
|
||||
mux.HandleFunc("GET /debug/logs", s.withGuest(s.handleDebugLogs))
|
||||
|
||||
Reference in New Issue
Block a user