361 lines
13 KiB
Go
361 lines
13 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Controller-driven escrow ceremony tests (v0.88.0). The runner seam returns canned SPIKE-SHAPED
|
|
// JSON — no sudo, no subprocess. The load-bearing assertions are the R custody rules: one-shot
|
|
// claim, TTL void, and R structurally absent from every status/snapshot payload.
|
|
|
|
const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"
|
|
|
|
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values).
|
|
func cannedCeremonyJSON(r string) string {
|
|
return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r)
|
|
}
|
|
|
|
// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
|
|
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
|
|
t.Helper()
|
|
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
|
srv.escrowCeremony = &EscrowCeremonyConfig{
|
|
PBSStorageID: "felhom-pbs",
|
|
HubConfigured: true,
|
|
DRConfigured: func() bool { return true },
|
|
}
|
|
srv.ceremonyRun = run
|
|
srv.escrowSudoCheck = func(context.Context) error { return nil }
|
|
srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
|
|
srv.statFile = func(string) bool { return true }
|
|
return srv
|
|
}
|
|
|
|
// okRunner returns the canned success output and counts invocations.
|
|
func okRunner(calls *atomic.Int32) ceremonyRunner {
|
|
return func(context.Context) ([]byte, []byte, int, error) {
|
|
if calls != nil {
|
|
calls.Add(1)
|
|
}
|
|
return []byte(cannedCeremonyJSON(testR)), []byte("info: ceremony fine\n"), 0, nil
|
|
}
|
|
}
|
|
|
|
// startAndWait POSTs /escrow/ceremony and waits for the detached job to finish.
|
|
func startAndWait(t *testing.T, srv *Server, h http.Handler) {
|
|
t.Helper()
|
|
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
|
|
t.Fatalf("start: got %d, want 202 (%s)", w.Code, w.Body.String())
|
|
}
|
|
select {
|
|
case <-srv.escrowDone:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("ceremony job did not finish")
|
|
}
|
|
}
|
|
|
|
// Scenario A/D happy path: run → done → claim ONCE (R delivered, no-store) → 410 on re-claim,
|
|
// holder zeroed. R never appears in the start or status payloads.
|
|
func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
|
var calls atomic.Int32
|
|
srv := newEscrowTestServer(t, okRunner(&calls))
|
|
h := srv.Handler()
|
|
|
|
startAndWait(t, srv, h)
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("runner called %d times, want 1", calls.Load())
|
|
}
|
|
|
|
// Status: done + claimable, summary populated, R ABSENT from the whole payload.
|
|
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
|
if st.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d", st.Code)
|
|
}
|
|
body := st.Body.String()
|
|
if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
|
|
t.Fatalf("status not done/claimable: %s", body)
|
|
}
|
|
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) {
|
|
t.Fatalf("summary fields missing: %s", body)
|
|
}
|
|
assertNoR(t, "status payload", body)
|
|
|
|
// First claim → 200 with EXACTLY the canned R + Cache-Control: no-store.
|
|
c1 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
|
if c1.Code != http.StatusOK {
|
|
t.Fatalf("claim 1: got %d (%s)", c1.Code, c1.Body.String())
|
|
}
|
|
if cc := c1.Header().Get("Cache-Control"); cc != "no-store" {
|
|
t.Fatalf("claim Cache-Control = %q, want no-store", cc)
|
|
}
|
|
var env struct {
|
|
Data struct {
|
|
RecoveryCode string `json:"recovery_code"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(c1.Body.Bytes(), &env); err != nil || env.Data.RecoveryCode != testR {
|
|
t.Fatalf("claim 1 recovery_code = %q, want the canned R", env.Data.RecoveryCode)
|
|
}
|
|
|
|
// The in-memory holder is gone the moment the claim returns.
|
|
srv.escrowMu.Lock()
|
|
holder := len(srv.escrowR)
|
|
srv.escrowMu.Unlock()
|
|
if holder != 0 {
|
|
t.Fatal("R holder survived the claim — the wipe-after-claim is missing")
|
|
}
|
|
|
|
// Second claim → 410 Gone, and no R anywhere in it.
|
|
c2 := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
|
if c2.Code != http.StatusGone {
|
|
t.Fatalf("claim 2: got %d, want 410", c2.Code)
|
|
}
|
|
assertNoR(t, "re-claim payload", c2.Body.String())
|
|
|
|
// Post-claim status: claimed, not claimable, still phase done.
|
|
st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
|
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) {
|
|
t.Fatalf("post-claim status wrong: %s", st2.Body.String())
|
|
}
|
|
}
|
|
|
|
// Scenario D TTL: an unclaimed R past the 10-min TTL is zeroed and the job flips to
|
|
// unclaimed_void; the claim answers 410. (The lazy s.now-driven path — the tested primary.)
|
|
func TestEscrowCeremony_TTLExpiryVoidsUnclaimedR(t *testing.T) {
|
|
srv := newEscrowTestServer(t, okRunner(nil))
|
|
cur := testNow
|
|
srv.now = func() time.Time { return cur }
|
|
h := srv.Handler()
|
|
|
|
startAndWait(t, srv, h)
|
|
|
|
cur = cur.Add(escrowClaimTTL + time.Minute) // jump past the TTL
|
|
|
|
c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
|
if c.Code != http.StatusGone {
|
|
t.Fatalf("claim after TTL: got %d, want 410", c.Code)
|
|
}
|
|
srv.escrowMu.Lock()
|
|
holder := len(srv.escrowR)
|
|
phase := srv.escrowJob.Phase
|
|
srv.escrowMu.Unlock()
|
|
if holder != 0 {
|
|
t.Fatal("R holder survived the TTL — the expiry wipe is missing")
|
|
}
|
|
if phase != escrowPhaseVoid {
|
|
t.Fatalf("phase after TTL = %q, want %q", phase, escrowPhaseVoid)
|
|
}
|
|
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
|
if !strings.Contains(st.Body.String(), `"phase":"unclaimed_void"`) {
|
|
t.Fatalf("status after TTL: %s", st.Body.String())
|
|
}
|
|
assertNoR(t, "void status payload", st.Body.String())
|
|
}
|
|
|
|
// Snapshot hygiene (the §10 mutation target): serialize the ENTIRE job struct — the thing every
|
|
// snapshot/status copy derives from — and prove the R substring cannot appear in it. Adding R
|
|
// (or the raw stdout) to escrowCeremonyJob makes this fail.
|
|
func TestEscrowCeremony_JobStructCannotCarryR(t *testing.T) {
|
|
srv := newEscrowTestServer(t, okRunner(nil))
|
|
h := srv.Handler()
|
|
startAndWait(t, srv, h)
|
|
|
|
srv.escrowMu.Lock()
|
|
raw, err := json.Marshal(srv.escrowJob)
|
|
srv.escrowMu.Unlock()
|
|
if err != nil {
|
|
t.Fatalf("marshal job: %v", err)
|
|
}
|
|
assertNoR(t, "serialized job struct", string(raw))
|
|
}
|
|
|
|
// Single-flight: a second start while one is RUNNING → 409, and the runner is not re-invoked.
|
|
func TestEscrowCeremony_SingleFlight409(t *testing.T) {
|
|
release := make(chan struct{})
|
|
var calls atomic.Int32
|
|
srv := newEscrowTestServer(t, func(ctx context.Context) ([]byte, []byte, int, error) {
|
|
calls.Add(1)
|
|
<-release
|
|
return []byte(cannedCeremonyJSON(testR)), nil, 0, nil
|
|
})
|
|
h := srv.Handler()
|
|
|
|
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusAccepted {
|
|
t.Fatalf("start 1: got %d", w.Code)
|
|
}
|
|
done := srv.escrowDone
|
|
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict {
|
|
t.Fatalf("start 2 while running: got %d, want 409", w.Code)
|
|
}
|
|
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"running"`) {
|
|
t.Fatalf("status while running: %s", st.Body.String())
|
|
}
|
|
close(release)
|
|
<-done
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("runner called %d times, want 1 (the 409 must not spawn)", calls.Load())
|
|
}
|
|
}
|
|
|
|
// A completed-but-unclaimed ceremony is SUPERSEDED by a re-run: the old R is zeroed before the
|
|
// new job takes the slot, and the eventual claim yields the NEW code only.
|
|
func TestEscrowCeremony_RerunSupersedesUnclaimedR(t *testing.T) {
|
|
const newR = "second-run-code-xxxx"
|
|
first := true
|
|
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
|
r := newR
|
|
if first {
|
|
r = testR
|
|
first = false
|
|
}
|
|
return []byte(cannedCeremonyJSON(r)), nil, 0, nil
|
|
})
|
|
h := srv.Handler()
|
|
|
|
startAndWait(t, srv, h) // run 1, R unclaimed
|
|
startAndWait(t, srv, h) // run 2 supersedes
|
|
|
|
c := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
|
if c.Code != http.StatusOK {
|
|
t.Fatalf("claim: got %d", c.Code)
|
|
}
|
|
if !strings.Contains(c.Body.String(), newR) {
|
|
t.Fatal("claim did not deliver the SECOND run's code")
|
|
}
|
|
assertNoR(t, "superseding claim payload", c.Body.String()) // the OLD R must be gone
|
|
}
|
|
|
|
// Failure paths: non-zero exit carries the stderr tail (log-clean per spike) into detail; stdout
|
|
// (secret-bearing) NEVER lands there. Unparseable stdout fails without echoing it. A failed job
|
|
// answers 409 on claim.
|
|
func TestEscrowCeremony_FailurePaths(t *testing.T) {
|
|
t.Run("exit nonzero", func(t *testing.T) {
|
|
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
|
return []byte("half-a-secret-" + testR), []byte("selftest=escrow-create: PBS key not found"), 1, fmt.Errorf("exit status 1")
|
|
})
|
|
h := srv.Handler()
|
|
startAndWait(t, srv, h)
|
|
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
|
body := st.Body.String()
|
|
if !strings.Contains(body, `"phase":"failed"`) || !strings.Contains(body, "PBS key not found") {
|
|
t.Fatalf("failed status lacks the stderr tail: %s", body)
|
|
}
|
|
assertNoR(t, "failed status payload", body)
|
|
if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusConflict {
|
|
t.Fatalf("claim on failed: got %d, want 409", c.Code)
|
|
}
|
|
})
|
|
t.Run("unparseable stdout", func(t *testing.T) {
|
|
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
|
return []byte("=== human banner leaked " + testR + " ==="), nil, 0, nil
|
|
})
|
|
h := srv.Handler()
|
|
startAndWait(t, srv, h)
|
|
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
|
if !strings.Contains(st.Body.String(), `"phase":"failed"`) {
|
|
t.Fatalf("want failed on unparseable stdout: %s", st.Body.String())
|
|
}
|
|
assertNoR(t, "unparseable-stdout status", st.Body.String())
|
|
})
|
|
t.Run("wrong version", func(t *testing.T) {
|
|
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
|
return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil
|
|
})
|
|
h := srv.Handler()
|
|
startAndWait(t, srv, h)
|
|
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"failed"`) {
|
|
t.Fatalf("want failed on version mismatch: %s", st.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// Restart honesty: a fresh process (empty slot) answers "none" / 404 — the controller treats a
|
|
// lost job as void and re-runs (crash-safety is in-memory BY DESIGN).
|
|
func TestEscrowCeremony_FreshSlotIsNone(t *testing.T) {
|
|
srv := newEscrowTestServer(t, okRunner(nil))
|
|
h := srv.Handler()
|
|
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", ""); !strings.Contains(st.Body.String(), `"phase":"none"`) {
|
|
t.Fatalf("fresh status: %s", st.Body.String())
|
|
}
|
|
if c := do(t, h, "POST", "/escrow/ceremony/claim", "A", ""); c.Code != http.StatusNotFound {
|
|
t.Fatalf("fresh claim: got %d, want 404", c.Code)
|
|
}
|
|
}
|
|
|
|
// Not-configured servers refuse all four routes (the Options.EscrowCeremony nil case).
|
|
func TestEscrowCeremony_NotConfigured(t *testing.T) {
|
|
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
|
h := srv.Handler()
|
|
for _, probe := range []struct{ method, path string }{
|
|
{"GET", "/escrow/preflight"}, {"POST", "/escrow/ceremony"},
|
|
{"GET", "/escrow/ceremony/status"}, {"POST", "/escrow/ceremony/claim"},
|
|
} {
|
|
if w := do(t, h, probe.method, probe.path, "A", ""); w.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("%s %s: got %d, want 503", probe.method, probe.path, w.Code)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Preflight: the all-green shape, the missing-grant red row, and the staged-secret item being
|
|
// INFORMATIONAL (its false never flips the aggregate ok — the controller owns that decision).
|
|
func TestEscrowPreflight(t *testing.T) {
|
|
srv := newEscrowTestServer(t, okRunner(nil))
|
|
srv.statFile = func(string) bool { return false } // no staged secret
|
|
h := srv.Handler()
|
|
|
|
w := do(t, h, "GET", "/escrow/preflight", "A", "")
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("preflight: got %d", w.Code)
|
|
}
|
|
var env struct {
|
|
Data struct {
|
|
OK bool `json:"ok"`
|
|
Items []struct {
|
|
ID string `json:"id"`
|
|
OK bool `json:"ok"`
|
|
} `json:"items"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if !env.Data.OK {
|
|
t.Fatalf("aggregate ok=false despite only the informational staged_secret being red: %s", w.Body.String())
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, it := range env.Data.Items {
|
|
seen[it.ID] = it.OK
|
|
}
|
|
for _, id := range []string{"pbs_storage_id", "dr_tier", "age_binary", "hub_upload", "sudo_grant"} {
|
|
if !seen[id] {
|
|
t.Fatalf("item %s not ok (or missing): %s", id, w.Body.String())
|
|
}
|
|
}
|
|
if ok, present := seen["staged_secret"]; !present || ok {
|
|
t.Fatalf("staged_secret should be present and false: %s", w.Body.String())
|
|
}
|
|
|
|
// Missing sudo grant → its row red AND the aggregate red (it is blocking).
|
|
srv.escrowSudoCheck = func(context.Context) error { return fmt.Errorf("denied") }
|
|
w2 := do(t, h, "GET", "/escrow/preflight", "A", "")
|
|
if !strings.Contains(w2.Body.String(), `"ok":false`) || !strings.Contains(w2.Body.String(), "FELHOM_ESCROW") {
|
|
t.Fatalf("missing grant should be a red, named row: %s", w2.Body.String())
|
|
}
|
|
}
|
|
|
|
// assertNoR fails the test if any fragment of the canned R appears in body — the R-handling
|
|
// absolute (§9 rule 4) checked at every non-claim surface.
|
|
func assertNoR(t *testing.T, where, body string) {
|
|
t.Helper()
|
|
if strings.Contains(body, testR) || strings.Contains(body, "canary-alpha") {
|
|
t.Fatalf("R leaked into %s: %s", where, body)
|
|
}
|
|
}
|