R-1254 (ceremony half, HUNT-3 H2): after a failed escrow upload, ask the hub which package it holds before wording the result

The hub holds the new package -> success; another package -> 502 'previous code still works' (now true);
cannot be asked -> 504, no promise. Tests TestR1254_* red before the fix. Commit only; the agent release needs the vouch.

internal/localapi/escrow_ceremony.go

@@ -72,12 +72,56 @@ type EscrowCeremonyConfig struct {
 	// host's hub record — called by the claim, BEFORE the code is handed out (R-978). nil → a claim of
 	// a pending package is refused (503): a code must never be handed out for a package the hub lacks.
 	Upload func(ctx context.Context, pkg []byte) error
+	// HubHolds (R-1254 H2) asks the hub whether its CURRENT package is exactly pkg — asked after a failed upload,
+	// whose answer may have been lost AFTER the hub stored it. nil or an error → the outcome is unknown.
+	HubHolds func(ctx context.Context, pkg []byte) (bool, error)
 }
 
 // escrowUploadTimeout bounds the claim-time package upload (one small PUT; the subprocess's whole
 // ceremony incl. its upload was measured at ~2.4 s).
 const escrowUploadTimeout = 30 * time.Second
 
+// escrowHoldTimeout bounds the hub question after a failed upload (R-1254 H2).
+const escrowHoldTimeout = 15 * time.Second
+
+// uploadOutcome is what a claim or confirm upload did, as the HUB tells it (R-1254 H2): an upload whose answer was lost
+// or late may have been stored all the same.
+type uploadOutcome int
+
+const (
+	uploadStored    uploadOutcome = iota // the hub holds the new package
+	uploadNotStored                      // the hub answered and holds another package: the previous code still works
+	uploadUnknown                        // the hub cannot be asked: nothing may be promised about either code
+)
+
+// escrowUnknownOutcomeMsg is the answer when the hub cannot say which package it holds — never „the previous code
+// still works", which is false when the lost answer came after the store.
+const escrowUnknownOutcomeMsg = "the hub did not answer in time — it is not known whether it stored the new recovery package; try again (until a try succeeds, do not rely on the previous code alone)"
+
+// outcomeAfterFailedUpload asks the hub whether it holds pkg after Upload returned an error. Called with the slot
+// marked claiming (escrowClaiming), outside escrowMu — it is a network call.
+func (s *Server) outcomeAfterFailedUpload(pkg []byte) uploadOutcome {
+	if s.escrowCeremony.HubHolds == nil {
+		return uploadUnknown
+	}
+	base := s.baseCtx
+	if base == nil {
+		base = context.Background()
+	}
+	ctx, cancel := context.WithTimeout(base, escrowHoldTimeout)
+	defer cancel()
+	held, err := s.escrowCeremony.HubHolds(ctx, pkg)
+	switch {
+	case err != nil:
+		s.logger.Warn("local-api: escrow — the hub cannot say which package it holds after a failed upload (R-1254)", "err", err)
+		return uploadUnknown
+	case held:
+		return uploadStored
+	default:
+		return uploadNotStored
+	}
+}
+
 // ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
 // parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
 // spike-shaped output; production is runCeremonySubprocess.
@@ -524,15 +568,30 @@ func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Reques
 		uctx, cancel := context.WithTimeout(base, escrowUploadTimeout)
 		err := s.escrowCeremony.Upload(uctx, pkg)
 		cancel()
+		outcome := uploadStored
+		if err != nil {
+			outcome = s.outcomeAfterFailedUpload(pkg) // R-1254 H2: the answer may be lost AFTER the store
+		}
 
 		s.escrowMu.Lock()
 		s.escrowClaiming = false
-		if err != nil {
+		switch outcome {
+		case uploadNotStored:
 			s.escrowMu.Unlock()
 			s.logger.Warn("local-api: escrow claim — package upload to the hub failed; code kept claimable, previous package stays current (R-978)",
 				"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
 			writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again")
 			return
+		case uploadUnknown:
+			s.escrowMu.Unlock()
+			s.logger.Warn("local-api: escrow claim — upload failed and the hub cannot say which package it holds; code kept claimable (R-1254)",
+				"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
+			writeErr(w, http.StatusGatewayTimeout, escrowUnknownOutcomeMsg)
+			return
+		}
+		if err != nil {
+			s.logger.Warn("local-api: escrow claim — the upload's answer was lost, but the hub holds the new package; handing out the code (R-1254)",
+				"vmid", vmid, "job_id", jobID, "err", err)
 		}
 		s.escrowPending = nil
 		if s.escrowJob != nil && s.escrowJob.JobID == jobID {
@@ -631,15 +690,30 @@ func (s *Server) handleEscrowCeremonyConfirm(w http.ResponseWriter, r *http.Requ
 	uctx, cancel := context.WithTimeout(base, escrowUploadTimeout)
 	err := s.escrowCeremony.Upload(uctx, pkg)
 	cancel()
+	outcome := uploadStored
+	if err != nil {
+		outcome = s.outcomeAfterFailedUpload(pkg) // R-1254 H2
+	}
 
 	s.escrowMu.Lock()
 	s.escrowClaiming = false
-	if err != nil {
+	switch outcome {
+	case uploadNotStored:
 		s.escrowMu.Unlock()
 		s.logger.Warn("local-api: escrow confirm — package upload to the hub failed; still waiting, previous package stays current (R-1151)",
 			"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
 		writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again")
 		return
+	case uploadUnknown:
+		s.escrowMu.Unlock()
+		s.logger.Warn("local-api: escrow confirm — upload failed and the hub cannot say which package it holds; still waiting (R-1254)",
+			"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
+		writeErr(w, http.StatusGatewayTimeout, escrowUnknownOutcomeMsg)
+		return
+	}
+	if err != nil {
+		s.logger.Warn("local-api: escrow confirm — the upload's answer was lost, but the hub holds the new package (R-1254)",
+			"vmid", vmid, "job_id", jobID, "err", err)
 	}
 	s.escrowPending = nil
 	s.wipeEscrowConfirmLocked()