package localapi import ( "context" "encoding/json" "fmt" "net/http" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" ) // Controller-driven escrow ceremony tests (v0.88.0). The runner seam returns canned SPIKE-SHAPED // JSON — no sudo, no subprocess. The load-bearing assertions are the R custody rules: one-shot // claim, TTL void, and R structurally absent from every status/snapshot payload. const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india" // cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values). func cannedCeremonyJSON(r string) string { return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r) } // newEscrowTestServer builds a server with the ceremony configured and every seam faked. func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server { t.Helper() srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) srv.escrowCeremony = &EscrowCeremonyConfig{ PBSStorageID: "felhom-pbs", HubConfigured: true, DRConfigured: func() bool { return true }, } srv.ceremonyRun = run srv.escrowSudoCheck = func(context.Context) error { return nil } srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil } srv.statFile = func(string) bool { return true } return srv } // okRunner returns the canned success output and counts invocations. func okRunner(calls *atomic.Int32) ceremonyRunner { return func(context.Context) ([]byte, []byte, int, error) { if calls != nil { calls.Add(1) } return []byte(cannedCeremonyJSON(testR)), []byte("info: ceremony fine\n"), 0, nil } } // startAndWait POSTs /escrow/ceremony and waits for the detached job to finish. func startAndWait(t *testing.T, srv *Server, h http.Handler) { t.Helper() if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted { t.Fatalf("start: got %d, want 202 (%s)", w.Code, w.Body.String()) } select { case <-srv.escrowDone: case <-time.After(5 * time.Second): t.Fatal("ceremony job did not finish") } } // TestEscrowPreflight_PBSStorageIDLiveReload pins the v0.89.0 live-reload: the pbsdr bridge seeds // escrow.pbs_storage_id into agent.json on DR convergence; the preflight must see that seed IN THE // SAME PROCESS (no restart). The daemon-start snapshot (cfg.PBSStorageID) is empty; only the // late-bound CurrentPBSStorageID resolver (re-reads disk) reflects the seed. RED-PROOF: the pre-fix // preflight read cfg.PBSStorageID directly and ignored the resolver, so step 3 stays red → FAIL. func TestEscrowPreflight_PBSStorageIDLiveReload(t *testing.T) { dir := t.TempDir() cfgPath := filepath.Join(dir, "agent.json") // Pre-convergence: agent.json has no escrow.pbs_storage_id. if err := os.WriteFile(cfgPath, []byte(`{"escrow":{"posture":"zero_knowledge"}}`), 0o600); err != nil { t.Fatalf("seed config: %v", err) } readCurrent := func() string { raw, err := os.ReadFile(cfgPath) if err != nil { return "" } var doc struct { Escrow struct { PBSStorageID string `json:"pbs_storage_id"` } `json:"escrow"` } _ = json.Unmarshal(raw, &doc) return doc.Escrow.PBSStorageID } srv := newEscrowTestServer(t, okRunner(nil)) srv.escrowCeremony.PBSStorageID = "" // daemon-start snapshot: empty (the pre-fix source) srv.escrowCeremony.CurrentPBSStorageID = readCurrent // live resolver → current disk state h := srv.Handler() rowOK := func(t *testing.T) (bool, string) { t.Helper() w := do(t, h, "GET", "/escrow/preflight", "A", "") if w.Code != http.StatusOK { t.Fatalf("preflight status %d: %s", w.Code, w.Body.String()) } var resp struct { Data struct { Items []struct { ID string `json:"id"` OK bool `json:"ok"` Detail string `json:"detail"` } `json:"items"` } `json:"data"` } if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatalf("decode preflight: %v (%s)", err, w.Body.String()) } for _, it := range resp.Data.Items { if it.ID == "pbs_storage_id" { return it.OK, it.Detail } } t.Fatal("pbs_storage_id row missing from preflight") return false, "" } // 1. Before convergence: the row is RED. if ok, _ := rowOK(t); ok { t.Fatal("pbs_storage_id row is green before any seed") } // 2. Convergence seeds the id IN-PROCESS (simulating pbsdr finishConverged → seedEscrowStorageID). if err := os.WriteFile(cfgPath, []byte(`{"escrow":{"posture":"zero_knowledge","pbs_storage_id":"felhom-offsite"}}`), 0o600); err != nil { t.Fatalf("seed after convergence: %v", err) } // 3. Same process, NO restart: the row flips GREEN and reports the seeded id. ok, detail := rowOK(t) if !ok { t.Fatal("pbs_storage_id row still red after the in-process seed — the preflight is not live-reloading (restart-only)") } if detail != "felhom-offsite" { t.Fatalf("pbs_storage_id detail = %q, want the seeded id felhom-offsite", detail) } } // Scenario A/D happy path: run → done → claim ONCE (R delivered, no-store) → 410 on re-claim, // holder zeroed. R never appears in the start or status payloads. func TestEscrowCeremony_OneShotClaim(t *testing.T) { var calls atomic.Int32 srv := newEscrowTestServer(t, okRunner(&calls)) h := srv.Handler() startAndWait(t, srv, h) if calls.Load() != 1 { t.Fatalf("runner called %d times, want 1", calls.Load()) } // Status: done + claimable, summary populated, R ABSENT from the whole payload. st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") if st.Code != http.StatusOK { t.Fatalf("status: got %d", st.Code) } body := st.Body.String() if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) { t.Fatalf("status not done/claimable: %s", body) } if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) { t.Fatalf("summary fields missing: %s", body) } assertNoR(t, "status payload", body) // First claim → 200 with EXACTLY the canned R + Cache-Control: no-store. c1 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") if c1.Code != http.StatusOK { t.Fatalf("claim 1: got %d (%s)", c1.Code, c1.Body.String()) } if cc := c1.Header().Get("Cache-Control"); cc != "no-store" { t.Fatalf("claim Cache-Control = %q, want no-store", cc) } var env struct { Data struct { RecoveryCode string `json:"recovery_code"` } `json:"data"` } if err := json.Unmarshal(c1.Body.Bytes(), &env); err != nil || env.Data.RecoveryCode != testR { t.Fatalf("claim 1 recovery_code = %q, want the canned R", env.Data.RecoveryCode) } // The in-memory holder is gone the moment the claim returns. srv.escrowMu.Lock() holder := len(srv.escrowR) srv.escrowMu.Unlock() if holder != 0 { t.Fatal("R holder survived the claim — the wipe-after-claim is missing") } // Second claim → 410 Gone, and no R anywhere in it. c2 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") if c2.Code != http.StatusGone { t.Fatalf("claim 2: got %d, want 410", c2.Code) } assertNoR(t, "re-claim payload", c2.Body.String()) // Post-claim status: claimed, not claimable, still phase done. st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "") if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) { t.Fatalf("post-claim status wrong: %s", st2.Body.String()) } } // Scenario D TTL: an unclaimed R past the 10-min TTL is zeroed and the job flips to // unclaimed_void; the claim answers 410. (The lazy s.now-driven path — the tested primary.) func TestEscrowCeremony_TTLExpiryVoidsUnclaimedR(t *testing.T) { srv := newEscrowTestServer(t, okRunner(nil)) cur := testNow srv.now = func() time.Time { return cur } h := srv.Handler() startAndWait(t, srv, h) cur = cur.Add(escrowClaimTTL + time.Minute) // jump past the TTL c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") if c.Code != http.StatusGone { t.Fatalf("claim after TTL: got %d, want 410", c.Code) } srv.escrowMu.Lock() holder := len(srv.escrowR) phase := srv.escrowJob.Phase srv.escrowMu.Unlock() if holder != 0 { t.Fatal("R holder survived the TTL — the expiry wipe is missing") } if phase != escrowPhaseVoid { t.Fatalf("phase after TTL = %q, want %q", phase, escrowPhaseVoid) } st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") if !strings.Contains(st.Body.String(), `"phase":"unclaimed_void"`) { t.Fatalf("status after TTL: %s", st.Body.String()) } assertNoR(t, "void status payload", st.Body.String()) } // Snapshot hygiene (the §10 mutation target): serialize the ENTIRE job struct — the thing every // snapshot/status copy derives from — and prove the R substring cannot appear in it. Adding R // (or the raw stdout) to escrowCeremonyJob makes this fail. func TestEscrowCeremony_JobStructCannotCarryR(t *testing.T) { srv := newEscrowTestServer(t, okRunner(nil)) h := srv.Handler() startAndWait(t, srv, h) srv.escrowMu.Lock() raw, err := json.Marshal(srv.escrowJob) srv.escrowMu.Unlock() if err != nil { t.Fatalf("marshal job: %v", err) } assertNoR(t, "serialized job struct", string(raw)) } // Single-flight: a second start while one is RUNNING → 409, and the runner is not re-invoked. func TestEscrowCeremony_SingleFlight409(t *testing.T) { release := make(chan struct{}) var calls atomic.Int32 srv := newEscrowTestServer(t, func(ctx context.Context) ([]byte, []byte, int, error) { calls.Add(1) <-release return []byte(cannedCeremonyJSON(testR)), nil, 0, nil }) h := srv.Handler() if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted { t.Fatalf("start 1: got %d", w.Code) } done := srv.escrowDone if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict { t.Fatalf("start 2 while running: got %d, want 409", w.Code) } if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"running"`) { t.Fatalf("status while running: %s", st.Body.String()) } close(release) <-done if calls.Load() != 1 { t.Fatalf("runner called %d times, want 1 (the 409 must not spawn)", calls.Load()) } } // A completed-but-unclaimed ceremony is SUPERSEDED by a re-run: the old R is zeroed before the // new job takes the slot, and the eventual claim yields the NEW code only. func TestEscrowCeremony_RerunSupersedesUnclaimedR(t *testing.T) { const newR = "second-run-code-xxxx" first := true srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { r := newR if first { r = testR first = false } return []byte(cannedCeremonyJSON(r)), nil, 0, nil }) h := srv.Handler() startAndWait(t, srv, h) // run 1, R unclaimed startAndWait(t, srv, h) // run 2 supersedes c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "") if c.Code != http.StatusOK { t.Fatalf("claim: got %d", c.Code) } if !strings.Contains(c.Body.String(), newR) { t.Fatal("claim did not deliver the SECOND run's code") } assertNoR(t, "superseding claim payload", c.Body.String()) // the OLD R must be gone } // Failure paths: non-zero exit carries the stderr tail (log-clean per spike) into detail; stdout // (secret-bearing) NEVER lands there. Unparseable stdout fails without echoing it. A failed job // answers 409 on claim. func TestEscrowCeremony_FailurePaths(t *testing.T) { t.Run("exit nonzero", func(t *testing.T) { srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { return []byte("half-a-secret-" + testR), []byte("selftest=escrow-create: PBS key not found"), 1, fmt.Errorf("exit status 1") }) h := srv.Handler() startAndWait(t, srv, h) st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") body := st.Body.String() if !strings.Contains(body, `"phase":"failed"`) || !strings.Contains(body, "PBS key not found") { t.Fatalf("failed status lacks the stderr tail: %s", body) } assertNoR(t, "failed status payload", body) if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusConflict { t.Fatalf("claim on failed: got %d, want 409", c.Code) } }) t.Run("unparseable stdout", func(t *testing.T) { srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { return []byte("=== human banner leaked " + testR + " ==="), nil, 0, nil }) h := srv.Handler() startAndWait(t, srv, h) st := do(t, h, "GET", "/escrow/ceremony/status", "A", "") if !strings.Contains(st.Body.String(), `"phase":"failed"`) { t.Fatalf("want failed on unparseable stdout: %s", st.Body.String()) } assertNoR(t, "unparseable-stdout status", st.Body.String()) }) t.Run("wrong version", func(t *testing.T) { srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) { return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil }) h := srv.Handler() startAndWait(t, srv, h) if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"failed"`) { t.Fatalf("want failed on version mismatch: %s", st.Body.String()) } }) } // Restart honesty: a fresh process (empty slot) answers "none" / 404 — the controller treats a // lost job as void and re-runs (crash-safety is in-memory BY DESIGN). func TestEscrowCeremony_FreshSlotIsNone(t *testing.T) { srv := newEscrowTestServer(t, okRunner(nil)) h := srv.Handler() if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"none"`) { t.Fatalf("fresh status: %s", st.Body.String()) } if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusNotFound { t.Fatalf("fresh claim: got %d, want 404", c.Code) } } // Not-configured servers refuse all four routes (the Options.EscrowCeremony nil case). func TestEscrowCeremony_NotConfigured(t *testing.T) { srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) h := srv.Handler() for _, probe := range []struct{ method, path string }{ {"GET", "/escrow/preflight"}, {"POST", "/escrow/ceremony"}, {"GET", "/escrow/ceremony/status"}, {"POST", "/escrow/ceremony/claim"}, } { if w := do(t, h, probe.method, probe.path, "A", ""); w.Code != http.StatusServiceUnavailable { t.Fatalf("%s %s: got %d, want 503", probe.method, probe.path, w.Code) } } } // Preflight: the all-green shape, the missing-grant red row, and the staged-secret item being // INFORMATIONAL (its false never flips the aggregate ok — the controller owns that decision). func TestEscrowPreflight(t *testing.T) { srv := newEscrowTestServer(t, okRunner(nil)) srv.statFile = func(string) bool { return false } // no staged secret h := srv.Handler() w := do(t, h, "GET", "/escrow/preflight", "A", "") if w.Code != http.StatusOK { t.Fatalf("preflight: got %d", w.Code) } var env struct { Data struct { OK bool `json:"ok"` Items []struct { ID string `json:"id"` OK bool `json:"ok"` } `json:"items"` } `json:"data"` } if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil { t.Fatalf("decode: %v", err) } if !env.Data.OK { t.Fatalf("aggregate ok=false despite only the informational staged_secret being red: %s", w.Body.String()) } seen := map[string]bool{} for _, it := range env.Data.Items { seen[it.ID] = it.OK } for _, id := range []string{"pbs_storage_id", "dr_tier", "age_binary", "hub_upload", "sudo_grant"} { if !seen[id] { t.Fatalf("item %s not ok (or missing): %s", id, w.Body.String()) } } if ok, present := seen["staged_secret"]; !present || ok { t.Fatalf("staged_secret should be present and false: %s", w.Body.String()) } // Missing sudo grant → its row red AND the aggregate red (it is blocking). srv.escrowSudoCheck = func(context.Context) error { return fmt.Errorf("denied") } w2 := do(t, h, "GET", "/escrow/preflight", "A", "") if !strings.Contains(w2.Body.String(), `"ok":false`) || !strings.Contains(w2.Body.String(), "FELHOM_ESCROW") { t.Fatalf("missing grant should be a red, named row: %s", w2.Body.String()) } } // assertNoR fails the test if any fragment of the canned R appears in body — the R-handling // absolute (§9 rule 4) checked at every non-claim surface. func assertNoR(t *testing.T, where, body string) { t.Helper() if strings.Contains(body, testR) || strings.Contains(body, "canary-alpha") { t.Fatalf("R leaked into %s: %s", where, body) } }