The new escrow package reaches the hub only when its code is claimed (R-978, unreleased)
The root ceremony uploaded the new package before the new code was claimed, and the hub overwrites the current package when the sealed secrets are unchanged: a customer who started the ceremony again and walked away kept a written-down code that opened nothing. Json mode (the FELHOM_ESCROW argv, unchanged) now returns the package (CeremonyOutput v2, PendingUpload); the daemon uploads it at POST /escrow/ceremony/claim before handing out the code. A failed upload is a 502 with the code still claimable; TTL and a new start wait for an in-flight claim upload; no uploader is a 503. Text mode still uploads at once. No sudoers change. Tests TestEscrowCeremony_R978_* (localapi) and the cmd-side defer/delivery/wiring tests, red-proved.
This commit is contained in:
admin committed
1 parent
f7b7308c4d
commit
8561dc8a5c
9 files changed
+592
-36
No files matched your search
@@ -24,9 +24,16 @@ import (
|
||||
// be structurally incapable of carrying R).
|
||||
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
||||
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
|
||||
// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
|
||||
// answer. No journal, deliberately — persistence is the one property R must never have.
|
||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R AND the package it opens,
|
||||
// which is SAFE (R-978: the package was never uploaded, so the hub still holds the previous one
|
||||
// and the customer's previous code still opens it; a re-run starts over). Status reporting "none"
|
||||
// after a restart is the honest answer. No journal, deliberately — persistence is the one
|
||||
// property R must never have.
|
||||
// - R-978: the NEW package reaches the hub only at the claim, and the code is handed out only after
|
||||
// the hub took it. Until then the previous package stays current — so a ceremony that is started
|
||||
// and abandoned (never claimed, superseded, TTL, restart) leaves the customer's written-down code
|
||||
// working. The TTL and a new start both wait while a claim's upload is in flight, so a package is
|
||||
// never stored without its code being handed out by that same claim.
|
||||
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
||||
// failure diagnostics only.
|
||||
|
||||
@@ -50,8 +57,16 @@ type EscrowCeremonyConfig struct {
|
||||
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
||||
// late-bound). nil → reported not-applied.
|
||||
DRConfigured func() bool
|
||||
// Upload sends a ceremony's package (CeremonyOutput.PendingUpload, the hub upload body) to this
|
||||
// host's hub record — called by the claim, BEFORE the code is handed out (R-978). nil → a claim of
|
||||
// a pending package is refused (503): a code must never be handed out for a package the hub lacks.
|
||||
Upload func(ctx context.Context, pkg []byte) error
|
||||
}
|
||||
|
||||
// escrowUploadTimeout bounds the claim-time package upload (one small PUT; the subprocess's whole
|
||||
// ceremony incl. its upload was measured at ~2.4 s).
|
||||
const escrowUploadTimeout = 30 * time.Second
|
||||
|
||||
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
||||
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
||||
// spike-shaped output; production is runCeremonySubprocess.
|
||||
@@ -129,6 +144,9 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
||||
return false
|
||||
}
|
||||
if s.escrowClaiming {
|
||||
return false // R-978: a claim is uploading this job's package — its code must reach the claimer
|
||||
}
|
||||
s.wipeEscrowRLocked()
|
||||
cp := *job
|
||||
s.escrowJob = &cp
|
||||
@@ -137,7 +155,7 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
||||
|
||||
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
||||
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string, pending []byte) {
|
||||
s.escrowMu.Lock()
|
||||
defer s.escrowMu.Unlock()
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
@@ -145,6 +163,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
s.escrowJob = &cp
|
||||
if job.Phase == escrowPhaseDone && r != "" {
|
||||
s.escrowR = []byte(r)
|
||||
s.escrowPending = append([]byte(nil), pending...)
|
||||
s.escrowRClaimed = false
|
||||
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
||||
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
||||
@@ -164,6 +183,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
||||
func (s *Server) wipeEscrowRLocked() {
|
||||
zeroBytes(s.escrowR)
|
||||
s.escrowR = nil
|
||||
s.escrowPending = nil // R-978: the package goes with its code — never uploaded once the code is gone
|
||||
}
|
||||
|
||||
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
||||
@@ -172,6 +192,9 @@ func (s *Server) expireEscrowRLocked() {
|
||||
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
||||
return
|
||||
}
|
||||
if s.escrowClaiming {
|
||||
return // R-978: the claim is uploading the package; voiding now would store a package nobody gets the code to
|
||||
}
|
||||
if len(s.escrowR) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -217,7 +240,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !s.tryStartEscrowCeremony(job) {
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
|
||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running, or its code is being claimed")
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
||||
@@ -247,7 +270,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
detail += " | stderr: " + tail
|
||||
}
|
||||
job.Phase, job.Detail = escrowPhaseFailed, detail
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
||||
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
||||
return
|
||||
@@ -255,24 +278,34 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
||||
var out escrow.CeremonyOutput
|
||||
if err := json.Unmarshal(stdout, &out); err != nil {
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
||||
return
|
||||
}
|
||||
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
||||
s.finishEscrowCeremony(job, "")
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
return
|
||||
}
|
||||
// R-978: the package must come back un-uploaded. An output that uploaded by itself (or carries no
|
||||
// package) would mean the previous package is already gone, or the new one can never be stored.
|
||||
if out.Uploaded || len(out.PendingUpload) == 0 {
|
||||
out.RecoveryCode = ""
|
||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output carries no pending package"
|
||||
s.finishEscrowCeremony(job, "", nil)
|
||||
s.logger.Warn("local-api: escrow ceremony output has no pending package — failed (R-978)", "job_id", job.JobID,
|
||||
"uploaded", out.Uploaded)
|
||||
return
|
||||
}
|
||||
job.Phase = escrowPhaseDone
|
||||
job.KeyFingerprint = out.KeyFingerprint
|
||||
job.EntropyBits = out.EntropyBits
|
||||
job.ResticPwSealed = out.ResticPwSealed
|
||||
job.Uploaded = out.Uploaded
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode)
|
||||
job.Uploaded = false // R-978: uploaded by the claim
|
||||
s.finishEscrowCeremony(job, out.RecoveryCode, out.PendingUpload)
|
||||
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
|
||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot); package uploads at the claim",
|
||||
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
||||
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
||||
}()
|
||||
@@ -339,11 +372,60 @@ func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Reques
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
||||
return
|
||||
case s.escrowClaiming:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "the recovery code is being claimed — try again in a moment")
|
||||
return
|
||||
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
||||
return
|
||||
}
|
||||
// R-978: store the NEW package on the hub first; only then hand out the code that opens it. Until this
|
||||
// upload succeeds the hub keeps the previous package, and the customer's previous code keeps working.
|
||||
if len(s.escrowPending) > 0 {
|
||||
if s.escrowCeremony.Upload == nil {
|
||||
s.escrowMu.Unlock()
|
||||
writeErr(w, http.StatusServiceUnavailable, "the recovery package cannot be stored on this host (no upload configured) — the previous code still works")
|
||||
return
|
||||
}
|
||||
pkg := s.escrowPending
|
||||
jobID := s.escrowJob.JobID
|
||||
s.escrowClaiming = true
|
||||
s.escrowMu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
uctx, cancel := context.WithTimeout(base, escrowUploadTimeout)
|
||||
err := s.escrowCeremony.Upload(uctx, pkg)
|
||||
cancel()
|
||||
|
||||
s.escrowMu.Lock()
|
||||
s.escrowClaiming = false
|
||||
if err != nil {
|
||||
s.escrowMu.Unlock()
|
||||
s.logger.Warn("local-api: escrow claim — package upload to the hub failed; code kept claimable, previous package stays current (R-978)",
|
||||
"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
|
||||
writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again")
|
||||
return
|
||||
}
|
||||
s.escrowPending = nil
|
||||
if s.escrowJob != nil && s.escrowJob.JobID == jobID {
|
||||
s.escrowJob.Uploaded = true
|
||||
s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
}
|
||||
s.logger.Info("local-api: escrow claim — package uploaded to the hub; handing out the code (R-978)",
|
||||
"vmid", vmid, "job_id", jobID, "duration_ms", time.Since(start).Milliseconds())
|
||||
if len(s.escrowR) == 0 { // cannot happen (TTL and supersede wait for the claim) — never hand out nothing
|
||||
s.escrowMu.Unlock()
|
||||
s.logger.Error("local-api: escrow claim — code vanished during the upload (invariant broken)", "job_id", jobID)
|
||||
writeErr(w, http.StatusInternalServerError, "the recovery code was lost during the claim — run a new ceremony")
|
||||
return
|
||||
}
|
||||
}
|
||||
recovery := string(s.escrowR)
|
||||
s.wipeEscrowRLocked()
|
||||
s.escrowRClaimed = true
|
||||
|
||||
Reference in new issue
Block a user