The new escrow package reaches the hub only when its code is claimed (R-978, unreleased)
The root ceremony uploaded the new package before the new code was claimed, and the hub overwrites the current package when the sealed secrets are unchanged: a customer who started the ceremony again and walked away kept a written-down code that opened nothing. Json mode (the FELHOM_ESCROW argv, unchanged) now returns the package (CeremonyOutput v2, PendingUpload); the daemon uploads it at POST /escrow/ceremony/claim before handing out the code. A failed upload is a 502 with the code still claimable; TTL and a new start wait for an in-flight claim upload; no uploader is a 503. Text mode still uploads at once. No sudoers change. Tests TestEscrowCeremony_R978_* (localapi) and the cmd-side defer/delivery/wiring tests, red-proved.
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
package escrow
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by
|
||||
// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API
|
||||
// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers
|
||||
@@ -43,7 +45,15 @@ type CeremonyOutput struct {
|
||||
IdentityBlobBytes int `json:"identity_blob_bytes"`
|
||||
ResticPwSealed bool `json:"restic_pw_sealed"`
|
||||
Uploaded bool `json:"uploaded"`
|
||||
// PendingUpload (version 2, R-978) is the hub upload body — the R-wrapped blobs (ciphertext), the key
|
||||
// fingerprint and the sealed password's hash — that json mode hands back INSTEAD of uploading. The agent
|
||||
// daemon uploads it only when the customer claims the new recovery code, so a ceremony that is started
|
||||
// and abandoned never replaces the package the customer's previous code opens. Not a secret without R,
|
||||
// but never logged. Uploaded is false whenever this is set.
|
||||
PendingUpload json.RawMessage `json:"pending_upload,omitempty"`
|
||||
}
|
||||
|
||||
// CeremonyOutputVersion is the current CeremonyOutput.Version value.
|
||||
const CeremonyOutputVersion = 1
|
||||
// CeremonyOutputVersion is the current CeremonyOutput.Version value. 2 (R-978): json mode no longer
|
||||
// uploads; it returns PendingUpload. The daemon refuses any other version (a binary/daemon mismatch
|
||||
// fails the ceremony instead of losing a package).
|
||||
const CeremonyOutputVersion = 2
|
||||
|
||||
@@ -24,9 +24,16 @@ import (
|
||||
// be structurally incapable of carrying R).
|
||||
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
||||
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
|
||||
// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
|
||||
// answer. No journal, deliberately — persistence is the one property R must never have.
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R AND the package it opens,
|
||||
// which is SAFE (R-978: the package was never uploaded, so the hub still holds the previous one
|
||||
// and the customer's previous code still opens it; a re-run starts over). Status reporting "none"
|
||||
// after a restart is the honest answer. No journal, deliberately — persistence is the one
|
||||
// property R must never have.
|
||||
// - R-978: the NEW package reaches the hub only at the claim, and the code is handed out only after
|
||||
// the hub took it. Until then the previous package stays current — so a ceremony that is started
|
||||
// and abandoned (never claimed, superseded, TTL, restart) leaves the customer's written-down code
|
||||
// working. The TTL and a new start both wait while a claim's upload is in flight, so a package is
|
||||
// never stored without its code being handed out by that same claim.
|
||||
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
||||
// failure diagnostics only.
|
||||
|
||||
@@ -50,8 +57,16 @@ type EscrowCeremonyConfig struct {
|
||||
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
||||
// late-bound). nil → reported not-applied.
|
||||
DRConfigured func() bool
|
||||
// Upload sends a ceremony's package (CeremonyOutput.PendingUpload, the hub upload body) to this
|
||||
// host's hub record — called by the claim, BEFORE the code is handed out (R-978). nil → a claim of
|
||||
// a pending package is refused (503): a code must never be handed out for a package the hub lacks.
|
||||
Upload func(ctx context.Context, pkg []byte) error
|
||||
}
|
||||
|
||||
// escrowUploadTimeout bounds the claim-time package upload (one small PUT; the subprocess's whole
|
||||
// ceremony incl. its upload was measured at ~2.4 s).
|
||||
const escrowUploadTimeout = 30 * time.Second
|
||||
|
||||
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
||||
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
||||
// spike-shaped output; production is runCeremonySubprocess.
|
||||
@@ -129,6 +144,9 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
||||
return false
|
||||
}
|
||||
if s.escrowClaiming {
|
||||
return false // R-978: a claim is uploading this job's package — its code must reach the claimer
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
@@ -137,7 +155,7 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
|
||||
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
||||
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string, pending []byte) {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
@@ -145,6 +163,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
s.escrowJob = &cp
|
||||
if job.Phase == escrowPhaseDone && r != "" {
|
||||
s.escrowR = []byte(r)
|
||||
s.escrowPending = append([]byte(nil), pending...)
|
||||
s.escrowRClaimed = false
|
||||
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
||||
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
||||
@@ -164,6 +183,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
func (s *Server) wipeEscrowRLocked() {
|
||||
zeroBytes(s.escrowR)
|
||||
s.escrowR = nil
|
||||
s.escrowPending = nil // R-978: the package goes with its code — never uploaded once the code is gone
|
||||
}
|
||||
|
||||
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
||||
@@ -172,6 +192,9 @@ func (s *Server) expireEscrowRLocked() {
|
||||
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
||||
return
|
||||
}
|
||||
if s.escrowClaiming {
|
||||
return // R-978: the claim is uploading the package; voiding now would store a package nobody gets the code to
|
||||
}
|
||||
if len(s.escrowR) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -217,7 +240,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !s.tryStartEscrowCeremony(job) {
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running, or its code is being claimed")
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
||||
@@ -247,7 +270,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
detail += " | stderr: " + tail
|
||||
}
|
||||
job.Phase, job.Detail = escrowPhaseFailed, detail
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
||||
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
||||
return
|
||||
@@ -255,24 +278,34 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
var out escrow.CeremonyOutput
|
||||
if err := json.Unmarshal(stdout, &out); err != nil {
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
||||
return
|
||||
}
|
||||
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
return
|
||||
}
|
||||
// R-978: the package must come back un-uploaded. An output that uploaded by itself (or carries no
|
||||
// package) would mean the previous package is already gone, or the new one can never be stored.
|
||||
if out.Uploaded || len(out.PendingUpload) == 0 {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output carries no pending package"
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony output has no pending package — failed (R-978)", "job_id", job.JobID,
|
||||
"uploaded", out.Uploaded)
|
||||
return
|
||||
}
|
||||
job.Phase = escrowPhaseDone
|
||||
job.KeyFingerprint = out.KeyFingerprint
|
||||
job.EntropyBits = out.EntropyBits
|
||||
job.ResticPwSealed = out.ResticPwSealed
|
||||
job.Uploaded = out.Uploaded
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode)
|
||||
job.Uploaded = false // R-978: uploaded by the claim
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode, out.PendingUpload)
|
||||
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot); package uploads at the claim",
|
||||
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
||||
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
||||
}()
|
||||
@@ -339,11 +372,60 @@ func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Reques
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
||||
return
|
||||
case s.escrowClaiming:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "the recovery code is being claimed — try again in a moment")
|
||||
return
|
||||
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
||||
return
|
||||
}
|
||||
// R-978: store the NEW package on the hub first; only then hand out the code that opens it. Until this
|
||||
// upload succeeds the hub keeps the previous package, and the customer's previous code keeps working.
|
||||
if len(s.escrowPending) > 0 {
|
||||
if s.escrowCeremony.Upload == nil {
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusServiceUnavailable, "the recovery package cannot be stored on this host (no upload configured) — the previous code still works")
|
||||
return
|
||||
}
|
||||
pkg := s.escrowPending
|
||||
jobID := s.escrowJob.JobID
|
||||
s.escrowClaiming = true
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
uctx, cancel := context.WithTimeout(base, escrowUploadTimeout)
|
||||
err := s.escrowCeremony.Upload(uctx, pkg)
|
||||
cancel()
|
||||
|
||||
s.escrowMu.Lock()
|
||||
s.escrowClaiming = false
|
||||
if err != nil {
|
||||
s.escrowMu.Unlock()
|
||||
s.logger.Warn("local-api: escrow claim — package upload to the hub failed; code kept claimable, previous package stays current (R-978)",
|
||||
"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
|
||||
writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again")
|
||||
return
|
||||
}
|
||||
s.escrowPending = nil
|
||||
if s.escrowJob != nil && s.escrowJob.JobID == jobID {
|
||||
s.escrowJob.Uploaded = true
|
||||
s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
}
|
||||
s.logger.Info("local-api: escrow claim — package uploaded to the hub; handing out the code (R-978)",
|
||||
"vmid", vmid, "job_id", jobID, "duration_ms", time.Since(start).Milliseconds())
|
||||
if len(s.escrowR) == 0 { // cannot happen (TTL and supersede wait for the claim) — never hand out nothing
|
||||
s.escrowMu.Unlock()
|
||||
s.logger.Error("local-api: escrow claim — code vanished during the upload (invariant broken)", "job_id", jobID)
|
||||
writeErr(w, http.StatusInternalServerError, "the recovery code was lost during the claim — run a new ceremony")
|
||||
return
|
||||
}
|
||||
}
|
||||
recovery := string(s.escrowR)
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowRClaimed = true
|
||||
|
||||
@@ -2,12 +2,14 @@ package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -19,25 +21,91 @@ import (
|
||||
|
||||
const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"
|
||||
|
||||
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values).
|
||||
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values;
|
||||
// version 2, R-978: the package comes back un-uploaded in pending_upload). The package is a MODEL of
|
||||
// the R-wrapped blob: it "opens" with exactly the code it was sealed under (fakeEscrowHub.opensWith).
|
||||
func cannedCeremonyJSON(r string) string {
|
||||
return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r)
|
||||
return fmt.Sprintf(`{"version":2,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":false,"pending_upload":%s}`,
|
||||
r, sealedPackage(r))
|
||||
}
|
||||
|
||||
// sealedPackage is the model hub upload body for a ceremony that minted code r.
|
||||
func sealedPackage(r string) string {
|
||||
return fmt.Sprintf(`{"blob_b64":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","posture":"zero_knowledge","created_at":"2026-06-10T12:00:00Z"}`,
|
||||
base64.StdEncoding.EncodeToString([]byte("sealed-under:"+r)))
|
||||
}
|
||||
|
||||
// fakeEscrowHub is the hub's current escrow package for this host (the PUT target). Upload replaces it,
|
||||
// as the hub does when the sealed secrets are unchanged (the R-978 case: no retained copy).
|
||||
type fakeEscrowHub struct {
|
||||
mu sync.Mutex
|
||||
current []byte
|
||||
uploads int
|
||||
err error // non-nil → the upload fails, nothing stored
|
||||
gate chan struct{} // non-nil → the upload blocks until closed
|
||||
entered chan struct{} // non-nil → closed when an upload starts
|
||||
}
|
||||
|
||||
func (f *fakeEscrowHub) upload(ctx context.Context, pkg []byte) error {
|
||||
if f.entered != nil {
|
||||
close(f.entered)
|
||||
f.entered = nil
|
||||
}
|
||||
if f.gate != nil {
|
||||
<-f.gate
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
}
|
||||
f.current = append([]byte(nil), pkg...)
|
||||
f.uploads++
|
||||
return nil
|
||||
}
|
||||
|
||||
// opensWith reports whether the hub's CURRENT package opens with code r — the customer's question.
|
||||
func (f *fakeEscrowHub) opensWith(r string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var pkg struct {
|
||||
BlobB64 string `json:"blob_b64"`
|
||||
}
|
||||
if json.Unmarshal(f.current, &pkg) != nil {
|
||||
return false
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(pkg.BlobB64)
|
||||
return err == nil && string(raw) == "sealed-under:"+r
|
||||
}
|
||||
|
||||
func (f *fakeEscrowHub) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.uploads
|
||||
}
|
||||
|
||||
// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
|
||||
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
|
||||
t.Helper()
|
||||
srv, _ := newEscrowTestServerHub(t, run, &fakeEscrowHub{})
|
||||
return srv
|
||||
}
|
||||
|
||||
// newEscrowTestServerHub is newEscrowTestServer with the hub's escrow record exposed (R-978).
|
||||
func newEscrowTestServerHub(t *testing.T, run ceremonyRunner, hub *fakeEscrowHub) (*Server, *fakeEscrowHub) {
|
||||
t.Helper()
|
||||
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
||||
srv.escrowCeremony = &EscrowCeremonyConfig{
|
||||
PBSStorageID: "felhom-pbs",
|
||||
HubConfigured: true,
|
||||
DRConfigured: func() bool { return true },
|
||||
Upload: hub.upload,
|
||||
}
|
||||
srv.ceremonyRun = run
|
||||
srv.escrowSudoCheck = func(context.Context) error { return nil }
|
||||
srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
|
||||
srv.statFile = func(string) bool { return true }
|
||||
return srv
|
||||
return srv, hub
|
||||
}
|
||||
|
||||
// okRunner returns the canned success output and counts invocations.
|
||||
@@ -160,8 +228,8 @@ func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
||||
if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
|
||||
t.Fatalf("status not done/claimable: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) {
|
||||
t.Fatalf("summary fields missing: %s", body)
|
||||
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":false`) {
|
||||
t.Fatalf("summary fields missing (R-978: not uploaded before the claim): %s", body)
|
||||
}
|
||||
assertNoR(t, "status payload", body)
|
||||
|
||||
@@ -199,7 +267,8 @@ func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
||||
|
||||
// Post-claim status: claimed, not claimable, still phase done.
|
||||
st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) {
|
||||
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) ||
|
||||
!strings.Contains(st2.Body.String(), `"uploaded":true`) {
|
||||
t.Fatalf("post-claim status wrong: %s", st2.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -344,7 +413,7 @@ func TestEscrowCeremony_FailurePaths(t *testing.T) {
|
||||
})
|
||||
t.Run("wrong version", func(t *testing.T) {
|
||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil
|
||||
return []byte(`{"version":1,"recovery_code":"` + testR + `","uploaded":true}`), nil, 0, nil
|
||||
})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-978 — starting the escrow ceremony again must not make the customer's written-down recovery code
|
||||
// useless. The hub overwrites the current package when the sealed secrets are unchanged, so the NEW
|
||||
// package may reach the hub only when its code is claimed. These tests ask the customer's question of
|
||||
// the hub's current package — „does my code still open it?" — not whether some function ran.
|
||||
|
||||
const (
|
||||
firstR = "first-ceremony-code-alpha"
|
||||
secondR = "second-ceremony-code-bravo"
|
||||
)
|
||||
|
||||
// sequenceRunner mints the given codes in order, one per ceremony.
|
||||
func sequenceRunner(codes ...string) ceremonyRunner {
|
||||
i := 0
|
||||
return func(context.Context) ([]byte, []byte, int, error) {
|
||||
r := codes[i]
|
||||
if i < len(codes)-1 {
|
||||
i++
|
||||
}
|
||||
return []byte(cannedCeremonyJSON(r)), nil, 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
func claimCode(t *testing.T, h http.Handler) (int, string) {
|
||||
t.Helper()
|
||||
w := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||
return w.Code, w.Body.String()
|
||||
}
|
||||
|
||||
// THE R-978 CASE. Ceremony 1 is claimed (the customer writes code 1 down). Ceremony 2 is started and
|
||||
// abandoned — three ways: its code expires unclaimed, a third start supersedes it, the agent restarts.
|
||||
// In every case the hub's current package must still open with code 1, and nothing was uploaded for 2.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-18): restore the pre-R-978 shape — the package uploaded when the
|
||||
// ceremony finishes (s.escrowCeremony.Upload(base, out.PendingUpload) in the job goroutine, as the root
|
||||
// subprocess did with `--upload`) and nothing left for the claim (finishEscrowCeremony(job, R, nil)) →
|
||||
// all three subtests FAIL with "the abandoned second ceremony replaced the package: code 1 no longer
|
||||
// opens it (uploads=2)" (uploads=3 for the superseding start). Restored.
|
||||
func TestEscrowCeremony_R978_AbandonedSecondCeremonyKeepsFirstCode(t *testing.T) {
|
||||
setup := func(t *testing.T) (*Server, *fakeEscrowHub, http.Handler, *time.Time) {
|
||||
srv, hub := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR, "third-code-charlie"), &fakeEscrowHub{})
|
||||
cur := testNow
|
||||
srv.now = func() time.Time { return cur }
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
if code, body := claimCode(t, h); code != http.StatusOK || !strings.Contains(body, firstR) {
|
||||
t.Fatalf("claim of ceremony 1: %d %s", code, body)
|
||||
}
|
||||
if !hub.opensWith(firstR) || hub.count() != 1 {
|
||||
t.Fatalf("after the first claim the hub must hold code 1's package (uploads=%d)", hub.count())
|
||||
}
|
||||
startAndWait(t, srv, h) // ceremony 2 — about to be abandoned
|
||||
return srv, hub, h, &cur
|
||||
}
|
||||
check := func(t *testing.T, hub *fakeEscrowHub) {
|
||||
t.Helper()
|
||||
if !hub.opensWith(firstR) {
|
||||
t.Fatalf("the abandoned second ceremony replaced the package: code 1 no longer opens it (uploads=%d)", hub.count())
|
||||
}
|
||||
if hub.opensWith(secondR) || hub.count() != 1 {
|
||||
t.Fatalf("a package for the never-claimed code 2 reached the hub (uploads=%d)", hub.count())
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("code 2 expires unclaimed", func(t *testing.T) {
|
||||
_, hub, h, cur := setup(t)
|
||||
*cur = cur.Add(escrowClaimTTL + time.Minute)
|
||||
if code, _ := claimCode(t, h); code != http.StatusGone {
|
||||
t.Fatalf("claim after TTL: got %d, want 410", code)
|
||||
}
|
||||
check(t, hub)
|
||||
})
|
||||
t.Run("a third start supersedes code 2", func(t *testing.T) {
|
||||
srv, hub, h, _ := setup(t)
|
||||
startAndWait(t, srv, h) // ceremony 3, also left unclaimed
|
||||
check(t, hub)
|
||||
})
|
||||
t.Run("the agent restarts before code 2 is claimed", func(t *testing.T) {
|
||||
_, hub, _, _ := setup(t)
|
||||
// A restart is a new process with an empty slot; only the hub's record survives it.
|
||||
srv2, _ := newEscrowTestServerHub(t, sequenceRunner("unused"), hub)
|
||||
if code, _ := claimCode(t, srv2.Handler()); code != http.StatusNotFound {
|
||||
t.Fatalf("claim after a restart: got %d, want 404", code)
|
||||
}
|
||||
check(t, hub)
|
||||
})
|
||||
}
|
||||
|
||||
// The positive control: CLAIMING code 2 is what replaces the package — the code handed out opens the
|
||||
// package the hub now holds, and the claim answers only after the hub took it.
|
||||
func TestEscrowCeremony_R978_ClaimStoresThePackageBeforeHandingOutTheCode(t *testing.T) {
|
||||
srv, hub := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR), &fakeEscrowHub{})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
if hub.count() != 0 {
|
||||
t.Fatalf("the package was uploaded before any claim (uploads=%d)", hub.count())
|
||||
}
|
||||
claimCode(t, h)
|
||||
startAndWait(t, srv, h)
|
||||
code, body := claimCode(t, h)
|
||||
if code != http.StatusOK || !strings.Contains(body, secondR) {
|
||||
t.Fatalf("claim of ceremony 2: %d %s", code, body)
|
||||
}
|
||||
if !hub.opensWith(secondR) || hub.count() != 2 {
|
||||
t.Fatalf("after claiming code 2 the hub must hold its package (uploads=%d, opens with 2: %v)", hub.count(), hub.opensWith(secondR))
|
||||
}
|
||||
}
|
||||
|
||||
// A failed upload hands out NO code, keeps the code claimable and the previous package current; a retry
|
||||
// that reaches the hub succeeds.
|
||||
func TestEscrowCeremony_R978_UploadFailureKeepsTheCodeClaimable(t *testing.T) {
|
||||
hub := &fakeEscrowHub{err: errors.New("hub returned HTTP 502")}
|
||||
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR), hub)
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
|
||||
code, body := claimCode(t, h)
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("claim with the hub failing: got %d, want 502 (%s)", code, body)
|
||||
}
|
||||
assertNoRIn(t, "failed-upload claim payload", body, firstR)
|
||||
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String(); !strings.Contains(st, `"claimable":true`) || !strings.Contains(st, `"uploaded":false`) {
|
||||
t.Fatalf("after a failed upload the code must stay claimable and un-uploaded: %s", st)
|
||||
}
|
||||
|
||||
hub.mu.Lock()
|
||||
hub.err = nil
|
||||
hub.mu.Unlock()
|
||||
code, body = claimCode(t, h)
|
||||
if code != http.StatusOK || !strings.Contains(body, firstR) || !hub.opensWith(firstR) {
|
||||
t.Fatalf("retry after the hub recovered: %d %s (opens: %v)", code, body, hub.opensWith(firstR))
|
||||
}
|
||||
}
|
||||
|
||||
// While the claim's upload is in flight, neither the TTL nor a new start may take the code away — else
|
||||
// the hub would hold a package whose code nobody was given (the R-978 failure, reached the other way).
|
||||
func TestEscrowCeremony_R978_TTLAndRestartWaitForAnInFlightClaim(t *testing.T) {
|
||||
hub := &fakeEscrowHub{gate: make(chan struct{}), entered: make(chan struct{})}
|
||||
entered := hub.entered
|
||||
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR), hub)
|
||||
var mu = make(chan struct{}, 1)
|
||||
mu <- struct{}{}
|
||||
cur := testNow
|
||||
srv.now = func() time.Time { <-mu; defer func() { mu <- struct{}{} }(); return cur }
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
|
||||
type res struct {
|
||||
code int
|
||||
body string
|
||||
}
|
||||
done := make(chan res, 1)
|
||||
go func() { c, b := claimCode(t, h); done <- res{c, b} }()
|
||||
<-entered
|
||||
|
||||
<-mu
|
||||
cur = cur.Add(escrowClaimTTL + time.Minute) // the TTL passes mid-upload
|
||||
mu <- struct{}{}
|
||||
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String(); strings.Contains(st, `"phase":"unclaimed_void"`) {
|
||||
t.Fatalf("the TTL voided the code while its package was uploading: %s", st)
|
||||
}
|
||||
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict {
|
||||
t.Fatalf("a new start during the claim's upload: got %d, want 409", w.Code)
|
||||
}
|
||||
if code, _ := claimCode(t, h); code != http.StatusConflict {
|
||||
t.Fatalf("a second claim during the upload: got %d, want 409", code)
|
||||
}
|
||||
close(hub.gate)
|
||||
r := <-done
|
||||
if r.code != http.StatusOK || !strings.Contains(r.body, firstR) || !hub.opensWith(firstR) {
|
||||
t.Fatalf("the in-flight claim must deliver the code whose package it stored: %d %s", r.code, r.body)
|
||||
}
|
||||
}
|
||||
|
||||
// No upload configured → the claim refuses rather than hand out a code for a package the hub lacks.
|
||||
func TestEscrowCeremony_R978_NoUploaderRefusesTheClaim(t *testing.T) {
|
||||
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR), &fakeEscrowHub{})
|
||||
srv.escrowCeremony.Upload = nil
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
code, body := claimCode(t, h)
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("claim with no uploader: got %d, want 503 (%s)", code, body)
|
||||
}
|
||||
assertNoRIn(t, "no-uploader claim payload", body, firstR)
|
||||
}
|
||||
|
||||
// An output that uploaded by itself (the pre-R-978 shape) or carries no package fails the ceremony.
|
||||
func TestEscrowCeremony_R978_OutputWithoutPendingPackageFails(t *testing.T) {
|
||||
srv, _ := newEscrowTestServerHub(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||
return []byte(`{"version":2,"recovery_code":"` + firstR + `","uploaded":true}`), nil, 0, nil
|
||||
}, &fakeEscrowHub{})
|
||||
h := srv.Handler()
|
||||
startAndWait(t, srv, h)
|
||||
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String()
|
||||
if !strings.Contains(st, `"phase":"failed"`) || !strings.Contains(st, "no pending package") {
|
||||
t.Fatalf("want failed: %s", st)
|
||||
}
|
||||
if code, _ := claimCode(t, h); code != http.StatusConflict {
|
||||
t.Fatalf("claim on a failed ceremony: got %d, want 409", code)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoRIn(t *testing.T, where, body, r string) {
|
||||
t.Helper()
|
||||
if strings.Contains(body, r) {
|
||||
t.Fatalf("%s carries the recovery code", where)
|
||||
}
|
||||
}
|
||||
@@ -426,6 +426,8 @@ type Server struct {
|
||||
escrowMu sync.Mutex
|
||||
escrowJob *escrowCeremonyJob
|
||||
escrowR []byte
|
||||
escrowPending []byte // R-978: the ceremony's hub upload body, sent by the claim before R is handed out
|
||||
escrowClaiming bool // R-978: a claim is uploading escrowPending (TTL and a new start wait)
|
||||
escrowRClaimed bool
|
||||
escrowRExpiry time.Time
|
||||
escrowDone <-chan struct{} // closes when the detached job finishes (tests wait on it)
|
||||
|
||||
Reference in New Issue
Block a user