felhom-agent/internal/localapi/escrow_ceremony_test.go

package localapi

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

// Controller-driven escrow ceremony tests (v0.88.0). The runner seam returns canned SPIKE-SHAPED
// JSON — no sudo, no subprocess. The load-bearing assertions are the R custody rules: one-shot
// claim, TTL void, and R structurally absent from every status/snapshot payload.

const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"

// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values;
// version 2, R-978: the package comes back un-uploaded in pending_upload). The package is a MODEL of
// the R-wrapped blob: it "opens" with exactly the code it was sealed under (fakeEscrowHub.opensWith).
func cannedCeremonyJSON(r string) string {
	return fmt.Sprintf(`{"version":2,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":false,"pending_upload":%s}`,
		r, sealedPackage(r))
}

// sealedPackage is the model hub upload body for a ceremony that minted code r.
func sealedPackage(r string) string {
	return fmt.Sprintf(`{"blob_b64":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","posture":"zero_knowledge","created_at":"2026-06-10T12:00:00Z"}`,
		base64.StdEncoding.EncodeToString([]byte("sealed-under:"+r)))
}

// fakeEscrowHub is the hub's current escrow package for this host (the PUT target). Upload replaces it,
// as the hub does when the sealed secrets are unchanged (the R-978 case: no retained copy).
type fakeEscrowHub struct {
	mu      sync.Mutex
	current []byte
	uploads int
	err     error         // non-nil → the upload fails, nothing stored
	gate    chan struct{} // non-nil → the upload blocks until closed
	entered chan struct{} // non-nil → closed when an upload starts
}

func (f *fakeEscrowHub) upload(ctx context.Context, pkg []byte) error {
	if f.entered != nil {
		close(f.entered)
		f.entered = nil
	}
	if f.gate != nil {
		<-f.gate
	}
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.err != nil {
		return f.err
	}
	f.current = append([]byte(nil), pkg...)
	f.uploads++
	return nil
}

// opensWith reports whether the hub's CURRENT package opens with code r — the customer's question.
func (f *fakeEscrowHub) opensWith(r string) bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	var pkg struct {
		BlobB64 string `json:"blob_b64"`
	}
	if json.Unmarshal(f.current, &pkg) != nil {
		return false
	}
	raw, err := base64.StdEncoding.DecodeString(pkg.BlobB64)
	return err == nil && string(raw) == "sealed-under:"+r
}

func (f *fakeEscrowHub) count() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.uploads
}

// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
	t.Helper()
	srv, _ := newEscrowTestServerHub(t, run, &fakeEscrowHub{})
	return srv
}

// newEscrowTestServerHub is newEscrowTestServer with the hub's escrow record exposed (R-978).
func newEscrowTestServerHub(t *testing.T, run ceremonyRunner, hub *fakeEscrowHub) (*Server, *fakeEscrowHub) {
	t.Helper()
	srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
	srv.escrowCeremony = &EscrowCeremonyConfig{
		PBSStorageID:  "felhom-pbs",
		HubConfigured: true,
		DRConfigured:  func() bool { return true },
		Upload:        hub.upload,
	}
	srv.ceremonyRun = run
	srv.escrowSudoCheck = func(context.Context) error { return nil }
	srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
	srv.statFile = func(string) bool { return true }
	return srv, hub
}

// okRunner returns the canned success output and counts invocations.
func okRunner(calls *atomic.Int32) ceremonyRunner {
	return func(context.Context) ([]byte, []byte, int, error) {
		if calls != nil {
			calls.Add(1)
		}
		return []byte(cannedCeremonyJSON(testR)), []byte("info: ceremony fine\n"), 0, nil
	}
}

// startAndWait POSTs /escrow/ceremony and waits for the detached job to finish.
func startAndWait(t *testing.T, srv *Server, h http.Handler) {
	t.Helper()
	if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
		t.Fatalf("start: got %d, want 202 (%s)", w.Code, w.Body.String())
	}
	select {
	case <-srv.escrowDone:
	case <-time.After(5 * time.Second):
		t.Fatal("ceremony job did not finish")
	}
}

// TestEscrowPreflight_PBSStorageIDLiveReload pins the v0.89.0 live-reload: the pbsdr bridge seeds
// escrow.pbs_storage_id into agent.json on DR convergence; the preflight must see that seed IN THE
// SAME PROCESS (no restart). The daemon-start snapshot (cfg.PBSStorageID) is empty; only the
// late-bound CurrentPBSStorageID resolver (re-reads disk) reflects the seed. RED-PROOF: the pre-fix
// preflight read cfg.PBSStorageID directly and ignored the resolver, so step 3 stays red → FAIL.
func TestEscrowPreflight_PBSStorageIDLiveReload(t *testing.T) {
	dir := t.TempDir()
	cfgPath := filepath.Join(dir, "agent.json")
	// Pre-convergence: agent.json has no escrow.pbs_storage_id.
	if err := os.WriteFile(cfgPath, []byte(`{"escrow":{"posture":"zero_knowledge"}}`), 0o600); err != nil {
		t.Fatalf("seed config: %v", err)
	}
	readCurrent := func() string {
		raw, err := os.ReadFile(cfgPath)
		if err != nil {
			return ""
		}
		var doc struct {
			Escrow struct {
				PBSStorageID string `json:"pbs_storage_id"`
			} `json:"escrow"`
		}
		_ = json.Unmarshal(raw, &doc)
		return doc.Escrow.PBSStorageID
	}

	srv := newEscrowTestServer(t, okRunner(nil))
	srv.escrowCeremony.PBSStorageID = ""                 // daemon-start snapshot: empty (the pre-fix source)
	srv.escrowCeremony.CurrentPBSStorageID = readCurrent // live resolver → current disk state
	h := srv.Handler()

	rowOK := func(t *testing.T) (bool, string) {
		t.Helper()
		w := do(t, h, "GET", "/escrow/preflight", "A", "")
		if w.Code != http.StatusOK {
			t.Fatalf("preflight status %d: %s", w.Code, w.Body.String())
		}
		var resp struct {
			Data struct {
				Items []struct {
					ID     string `json:"id"`
					OK     bool   `json:"ok"`
					Detail string `json:"detail"`
				} `json:"items"`
			} `json:"data"`
		}
		if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
			t.Fatalf("decode preflight: %v (%s)", err, w.Body.String())
		}
		for _, it := range resp.Data.Items {
			if it.ID == "pbs_storage_id" {
				return it.OK, it.Detail
			}
		}
		t.Fatal("pbs_storage_id row missing from preflight")
		return false, ""
	}

	// 1. Before convergence: the row is RED.
	if ok, _ := rowOK(t); ok {
		t.Fatal("pbs_storage_id row is green before any seed")
	}
	// 2. Convergence seeds the id IN-PROCESS (simulating pbsdr finishConverged → seedEscrowStorageID).
	if err := os.WriteFile(cfgPath, []byte(`{"escrow":{"posture":"zero_knowledge","pbs_storage_id":"felhom-offsite"}}`), 0o600); err != nil {
		t.Fatalf("seed after convergence: %v", err)
	}
	// 3. Same process, NO restart: the row flips GREEN and reports the seeded id.
	ok, detail := rowOK(t)
	if !ok {
		t.Fatal("pbs_storage_id row still red after the in-process seed — the preflight is not live-reloading (restart-only)")
	}
	if detail != "felhom-offsite" {
		t.Fatalf("pbs_storage_id detail = %q, want the seeded id felhom-offsite", detail)
	}
}

// Scenario A/D happy path: run → done → claim ONCE (R delivered, no-store) → 410 on re-claim,
// holder zeroed. R never appears in the start or status payloads.
func TestEscrowCeremony_OneShotClaim(t *testing.T) {
	var calls atomic.Int32
	srv := newEscrowTestServer(t, okRunner(&calls))
	h := srv.Handler()

	startAndWait(t, srv, h)
	if calls.Load() != 1 {
		t.Fatalf("runner called %d times, want 1", calls.Load())
	}

	// Status: done + claimable, summary populated, R ABSENT from the whole payload.
	st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
	if st.Code != http.StatusOK {
		t.Fatalf("status: got %d", st.Code)
	}
	body := st.Body.String()
	if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
		t.Fatalf("status not done/claimable: %s", body)
	}
	if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":false`) {
		t.Fatalf("summary fields missing (R-978: not uploaded before the claim): %s", body)
	}
	assertNoR(t, "status payload", body)

	// First claim → 200 with EXACTLY the canned R + Cache-Control: no-store.
	c1 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
	if c1.Code != http.StatusOK {
		t.Fatalf("claim 1: got %d (%s)", c1.Code, c1.Body.String())
	}
	if cc := c1.Header().Get("Cache-Control"); cc != "no-store" {
		t.Fatalf("claim Cache-Control = %q, want no-store", cc)
	}
	var env struct {
		Data struct {
			RecoveryCode string `json:"recovery_code"`
		} `json:"data"`
	}
	if err := json.Unmarshal(c1.Body.Bytes(), &env); err != nil || env.Data.RecoveryCode != testR {
		t.Fatalf("claim 1 recovery_code = %q, want the canned R", env.Data.RecoveryCode)
	}

	// The in-memory holder is gone the moment the claim returns.
	srv.escrowMu.Lock()
	holder := len(srv.escrowR)
	srv.escrowMu.Unlock()
	if holder != 0 {
		t.Fatal("R holder survived the claim — the wipe-after-claim is missing")
	}

	// Second claim → 410 Gone, and no R anywhere in it.
	c2 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
	if c2.Code != http.StatusGone {
		t.Fatalf("claim 2: got %d, want 410", c2.Code)
	}
	assertNoR(t, "re-claim payload", c2.Body.String())

	// Post-claim status: claimed, not claimable, still phase done.
	st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
	if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) ||
		!strings.Contains(st2.Body.String(), `"uploaded":true`) {
		t.Fatalf("post-claim status wrong: %s", st2.Body.String())
	}
}

// Scenario D TTL: an unclaimed R past the 10-min TTL is zeroed and the job flips to
// unclaimed_void; the claim answers 410. (The lazy s.now-driven path — the tested primary.)
func TestEscrowCeremony_TTLExpiryVoidsUnclaimedR(t *testing.T) {
	srv := newEscrowTestServer(t, okRunner(nil))
	cur := testNow
	srv.now = func() time.Time { return cur }
	h := srv.Handler()

	startAndWait(t, srv, h)

	cur = cur.Add(escrowClaimTTL + time.Minute) // jump past the TTL

	c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
	if c.Code != http.StatusGone {
		t.Fatalf("claim after TTL: got %d, want 410", c.Code)
	}
	srv.escrowMu.Lock()
	holder := len(srv.escrowR)
	phase := srv.escrowJob.Phase
	srv.escrowMu.Unlock()
	if holder != 0 {
		t.Fatal("R holder survived the TTL — the expiry wipe is missing")
	}
	if phase != escrowPhaseVoid {
		t.Fatalf("phase after TTL = %q, want %q", phase, escrowPhaseVoid)
	}
	st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
	if !strings.Contains(st.Body.String(), `"phase":"unclaimed_void"`) {
		t.Fatalf("status after TTL: %s", st.Body.String())
	}
	assertNoR(t, "void status payload", st.Body.String())
}

// Snapshot hygiene (the §10 mutation target): serialize the ENTIRE job struct — the thing every
// snapshot/status copy derives from — and prove the R substring cannot appear in it. Adding R
// (or the raw stdout) to escrowCeremonyJob makes this fail.
func TestEscrowCeremony_JobStructCannotCarryR(t *testing.T) {
	srv := newEscrowTestServer(t, okRunner(nil))
	h := srv.Handler()
	startAndWait(t, srv, h)

	srv.escrowMu.Lock()
	raw, err := json.Marshal(srv.escrowJob)
	srv.escrowMu.Unlock()
	if err != nil {
		t.Fatalf("marshal job: %v", err)
	}
	assertNoR(t, "serialized job struct", string(raw))
}

// Single-flight: a second start while one is RUNNING → 409, and the runner is not re-invoked.
func TestEscrowCeremony_SingleFlight409(t *testing.T) {
	release := make(chan struct{})
	var calls atomic.Int32
	srv := newEscrowTestServer(t, func(ctx context.Context) ([]byte, []byte, int, error) {
		calls.Add(1)
		<-release
		return []byte(cannedCeremonyJSON(testR)), nil, 0, nil
	})
	h := srv.Handler()

	if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
		t.Fatalf("start 1: got %d", w.Code)
	}
	done := srv.escrowDone
	if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict {
		t.Fatalf("start 2 while running: got %d, want 409", w.Code)
	}
	if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"running"`) {
		t.Fatalf("status while running: %s", st.Body.String())
	}
	close(release)
	<-done
	if calls.Load() != 1 {
		t.Fatalf("runner called %d times, want 1 (the 409 must not spawn)", calls.Load())
	}
}

// A completed-but-unclaimed ceremony is SUPERSEDED by a re-run: the old R is zeroed before the
// new job takes the slot, and the eventual claim yields the NEW code only.
func TestEscrowCeremony_RerunSupersedesUnclaimedR(t *testing.T) {
	const newR = "second-run-code-xxxx"
	first := true
	srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
		r := newR
		if first {
			r = testR
			first = false
		}
		return []byte(cannedCeremonyJSON(r)), nil, 0, nil
	})
	h := srv.Handler()

	startAndWait(t, srv, h) // run 1, R unclaimed
	startAndWait(t, srv, h) // run 2 supersedes

	c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
	if c.Code != http.StatusOK {
		t.Fatalf("claim: got %d", c.Code)
	}
	if !strings.Contains(c.Body.String(), newR) {
		t.Fatal("claim did not deliver the SECOND run's code")
	}
	assertNoR(t, "superseding claim payload", c.Body.String()) // the OLD R must be gone
}

// Failure paths: non-zero exit carries the stderr tail (log-clean per spike) into detail; stdout
// (secret-bearing) NEVER lands there. Unparseable stdout fails without echoing it. A failed job
// answers 409 on claim.
func TestEscrowCeremony_FailurePaths(t *testing.T) {
	t.Run("exit nonzero", func(t *testing.T) {
		srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
			return []byte("half-a-secret-" + testR), []byte("selftest=escrow-create: PBS key not found"), 1, fmt.Errorf("exit status 1")
		})
		h := srv.Handler()
		startAndWait(t, srv, h)
		st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
		body := st.Body.String()
		if !strings.Contains(body, `"phase":"failed"`) || !strings.Contains(body, "PBS key not found") {
			t.Fatalf("failed status lacks the stderr tail: %s", body)
		}
		assertNoR(t, "failed status payload", body)
		if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusConflict {
			t.Fatalf("claim on failed: got %d, want 409", c.Code)
		}
	})
	t.Run("unparseable stdout", func(t *testing.T) {
		srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
			return []byte("=== human banner leaked " + testR + " ==="), nil, 0, nil
		})
		h := srv.Handler()
		startAndWait(t, srv, h)
		st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
		if !strings.Contains(st.Body.String(), `"phase":"failed"`) {
			t.Fatalf("want failed on unparseable stdout: %s", st.Body.String())
		}
		assertNoR(t, "unparseable-stdout status", st.Body.String())
	})
	t.Run("wrong version", func(t *testing.T) {
		srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
			return []byte(`{"version":1,"recovery_code":"` + testR + `","uploaded":true}`), nil, 0, nil
		})
		h := srv.Handler()
		startAndWait(t, srv, h)
		if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"failed"`) {
			t.Fatalf("want failed on version mismatch: %s", st.Body.String())
		}
	})
}

// Restart honesty: a fresh process (empty slot) answers "none" / 404 — the controller treats a
// lost job as void and re-runs (crash-safety is in-memory BY DESIGN).
func TestEscrowCeremony_FreshSlotIsNone(t *testing.T) {
	srv := newEscrowTestServer(t, okRunner(nil))
	h := srv.Handler()
	if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"none"`) {
		t.Fatalf("fresh status: %s", st.Body.String())
	}
	if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusNotFound {
		t.Fatalf("fresh claim: got %d, want 404", c.Code)
	}
}

// Not-configured servers refuse all four routes (the Options.EscrowCeremony nil case).
func TestEscrowCeremony_NotConfigured(t *testing.T) {
	srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
	h := srv.Handler()
	for _, probe := range []struct{ method, path string }{
		{"GET", "/escrow/preflight"}, {"POST", "/escrow/ceremony"},
		{"GET", "/escrow/ceremony/status"}, {"POST", "/escrow/ceremony/claim"},
	} {
		if w := do(t, h, probe.method, probe.path, "A", ""); w.Code != http.StatusServiceUnavailable {
			t.Fatalf("%s %s: got %d, want 503", probe.method, probe.path, w.Code)
		}
	}
}

// Preflight: the all-green shape, the missing-grant red row, and the staged-secret item being
// INFORMATIONAL (its false never flips the aggregate ok — the controller owns that decision).
func TestEscrowPreflight(t *testing.T) {
	srv := newEscrowTestServer(t, okRunner(nil))
	srv.statFile = func(string) bool { return false } // no staged secret
	h := srv.Handler()

	w := do(t, h, "GET", "/escrow/preflight", "A", "")
	if w.Code != http.StatusOK {
		t.Fatalf("preflight: got %d", w.Code)
	}
	var env struct {
		Data struct {
			OK    bool `json:"ok"`
			Items []struct {
				ID string `json:"id"`
				OK bool   `json:"ok"`
			} `json:"items"`
		} `json:"data"`
	}
	if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if !env.Data.OK {
		t.Fatalf("aggregate ok=false despite only the informational staged_secret being red: %s", w.Body.String())
	}
	seen := map[string]bool{}
	for _, it := range env.Data.Items {
		seen[it.ID] = it.OK
	}
	for _, id := range []string{"pbs_storage_id", "dr_tier", "age_binary", "hub_upload", "sudo_grant"} {
		if !seen[id] {
			t.Fatalf("item %s not ok (or missing): %s", id, w.Body.String())
		}
	}
	if ok, present := seen["staged_secret"]; !present || ok {
		t.Fatalf("staged_secret should be present and false: %s", w.Body.String())
	}

	// Missing sudo grant → its row red AND the aggregate red (it is blocking).
	srv.escrowSudoCheck = func(context.Context) error { return fmt.Errorf("denied") }
	w2 := do(t, h, "GET", "/escrow/preflight", "A", "")
	if !strings.Contains(w2.Body.String(), `"ok":false`) || !strings.Contains(w2.Body.String(), "FELHOM_ESCROW") {
		t.Fatalf("missing grant should be a red, named row: %s", w2.Body.String())
	}
}

// assertNoR fails the test if any fragment of the canned R appears in body — the R-handling
// absolute (§9 rule 4) checked at every non-claim surface.
func assertNoR(t *testing.T, where, body string) {
	t.Helper()
	if strings.Contains(body, testR) || strings.Contains(body, "canary-alpha") {
		t.Fatalf("R leaked into %s: %s", where, body)
	}
}