diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 1866aa5..030ba57 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,36 @@ +## v0.94.0 — a box can fetch its own sealed recovery package (2026-08-04, R-199 link 6) + +**Chain link 6 had no client.** The hub has served the identity blob since slice 10D, from +`handleReEnroll` / `handleGetRestoreDirective` — gated on operator-armed recovery mode and the global +key. Nothing in the agent, the hub UI, any script or any runbook ever called either. The only +documented retrieval was `sqlite3 … writefile(…)` by hand on a `kubectl cp`-ed database. + +**`GET /api/v1/hosts/{host_id}/escrow`** — the box-authenticated MIRROR of the PUT that put the blob +there. Self-scoped: a per-host key reads only its own; the global key may read any, the same asymmetry +the PUT has. A host with no sealed bundle gets `200 {present:false}` — a clean answer, because a 404 +is indistinguishable from an unknown host and a bare empty 200 from a zero-length blob, and neither +of those is what "no ceremony has run yet" means. + +**THE TRADE, RECORDED IN THE HANDLER RATHER THAN INFERRED.** Before this, obtaining the blob required +the OPERATOR to arm recovery mode. Now whoever controls a rebuilt box can obtain it with that box's +own credential. That is a real reduction in the number of parties required, and it is the whole of the +trade — they still cannot open it, because the hub has never held R and a wrong code fails closed at +age's scrypt KDF. **The mitigation is that the capability is audited rather than silent:** every +successful retrieval raises `escrow_blob_served` (warning, operator-only), recorded before the bytes +leave. A silent capability on this object is the shape the last two weeks were spent removing. + +`escrowSelfServiceRetrieval` is a single named constant — the §8.2/§8.3 decision point. Flipping it to +false re-imposes the recovery-mode requirement on this endpoint and changes nothing else, so the +operator can overrule the trade at the cost of a boolean rather than a redesign. + +**The operator-driven DR path is untouched** — same gate, same behaviour, pinned by a test that +exercises re-enroll and restore-directive with recovery mode off and on. Red-proofs observed: removing +the ownership check makes a cross-host read succeed (host A served host B's blob); removing the audit +record makes the retrieval silent. + +**Not in this release:** the customer-facing flow. No card, no form, no preview — those are designed on +ground that has been walked, and R-200/R-201 are that walk. + ## v0.93.0 — the retention keeps the key it was built to keep, and three things stop lying (2026-08-04, R-198/R-197/R-196/R-192) ### R-198 — the superseded-escrow retention was preserving the wrong key, and the ceremony was destroying the right one diff --git a/hub/internal/api/escrow_get_test.go b/hub/internal/api/escrow_get_test.go new file mode 100644 index 0000000..17e06a9 --- /dev/null +++ b/hub/internal/api/escrow_get_test.go @@ -0,0 +1,203 @@ +package api + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/notify" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-199 (hub v0.94.0) — the box-authenticated retrieval of a host's OWN sealed identity blob. +// This is a new surface on the most sensitive object in the system; these tests exist to pin the +// three properties that make it defensible: it is self-scoped, it is honest when there is nothing to +// serve, and it is never silent. + +func seedEscrowedHost(t *testing.T, st *store.Store, hostID, customerID, apiKey string, identity []byte) { + t.Helper() + if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: apiKey}); err != nil { + t.Fatal(err) + } + if _, _, err := st.SaveHostEscrow(hostID, []byte("k-escrow"), "fp", "zero_knowledge", "2026-08-04T11:00:00Z", "SHA"); err != nil { + t.Fatal(err) + } + if len(identity) > 0 { + if err := st.SaveHostDRBundle(hostID, identity, `{}`); err != nil { + t.Fatal(err) + } + } +} + +// The happy path: a host reads back exactly the bytes it uploaded, verbatim. +func TestEscrowGet_ServesOwnBlobVerbatim(t *testing.T) { + h, st, _ := newTestHandler(t) + identity := []byte("\x00\x01age-wrapped-identity-bundle\xff") + seedEscrowedHost(t, st, "h1", "c1", "HKEY", identity) + + rr := do(h, http.MethodGet, "/hosts/h1/escrow", "HKEY", "") + if rr.Code != http.StatusOK { + t.Fatalf("GET escrow = %d, want 200 (%s)", rr.Code, rr.Body.String()) + } + var out struct { + HostID string `json:"host_id"` + Present bool `json:"present"` + B64 string `json:"identity_escrow_b64"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if !out.Present || out.HostID != "h1" { + t.Fatalf("unexpected envelope: %+v", out) + } + got, err := base64.StdEncoding.DecodeString(out.B64) + if err != nil { + t.Fatal(err) + } + if string(got) != string(identity) { + t.Fatal("the served blob is not the stored blob — the hub must return ciphertext verbatim") + } +} + +// Scenario C — a box can fetch ONLY its own blob. +// RED-PROOF: delete the `!isGlobal && authHostID != pathHostID` check in handleHostEscrowGet → +// cross-host retrieval succeeds → this FAILS. Without that line, any host key is a fleet-wide reader +// of every customer's sealed bundle. +func TestEscrowGet_CrossHostRefused(t *testing.T) { + h, st, _ := newTestHandler(t) + seedEscrowedHost(t, st, "hostA", "custA", "KEY-A", []byte("A-identity")) + seedEscrowedHost(t, st, "hostB", "custB", "KEY-B", []byte("B-identity-SECRET")) + + rr := do(h, http.MethodGet, "/hosts/hostB/escrow", "KEY-A", "") + if rr.Code != http.StatusForbidden { + t.Fatalf("host A reading host B's blob = %d, want 403 — a cross-host read of a sealed bundle "+ + "must be impossible on every code path (body: %s)", rr.Code, rr.Body.String()) + } + if b := rr.Body.String(); len(b) > 0 && (contains(b, "B-identity") || contains(b, base64.StdEncoding.EncodeToString([]byte("B-identity-SECRET")))) { + t.Fatal("the refusal body leaked the other host's blob") + } + // Unauthenticated is refused too, and does not leak which hosts exist. + if rr := do(h, http.MethodGet, "/hosts/hostB/escrow", "", ""); rr.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated GET = %d, want 401", rr.Code) + } + // The global operator key MAY read any — the same asymmetry the PUT has. + if rr := do(h, http.MethodGet, "/hosts/hostB/escrow", globalKey, ""); rr.Code != http.StatusOK { + t.Fatalf("global key GET = %d, want 200", rr.Code) + } +} + +func contains(hay, needle string) bool { + return len(needle) > 0 && len(hay) >= len(needle) && (func() bool { + for i := 0; i+len(needle) <= len(hay); i++ { + if hay[i:i+len(needle)] == needle { + return true + } + } + return false + })() +} + +// Scenario D — a host with no sealed bundle gets a CLEAN answer: 200 with present=false. Not a 404 +// (indistinguishable from an unknown host), not an empty 200 without the flag (indistinguishable +// from a zero-length blob). Both of those read as a fault to a caller whose situation is simply +// "no ceremony has run yet". +func TestEscrowGet_NoBlobIsCleanNone(t *testing.T) { + h, st, _ := newTestHandler(t) + seedEscrowedHost(t, st, "h1", "c1", "HKEY", nil) // K-escrow only, no identity blob + + rr := do(h, http.MethodGet, "/hosts/h1/escrow", "HKEY", "") + if rr.Code != http.StatusOK { + t.Fatalf("no-blob GET = %d, want 200 with present=false", rr.Code) + } + var out struct { + Present bool `json:"present"` + B64 string `json:"identity_escrow_b64"` + } + json.Unmarshal(rr.Body.Bytes(), &out) + if out.Present || out.B64 != "" { + t.Fatalf("a host with no bundle must report present=false and no bytes, got %+v", out) + } + // An unknown host is a DIFFERENT answer — the two must not collapse into one. + if rr := do(h, http.MethodGet, "/hosts/nope/escrow", globalKey, ""); rr.Code != http.StatusNotFound { + t.Fatalf("unknown host = %d, want 404 (distinct from a known host with no bundle)", rr.Code) + } +} + +// Scenario E — every successful retrieval leaves a durable record naming the host. +// RED-PROOF: remove the SaveEvent call in handleHostEscrowGet → no event → this FAILS. A silent +// capability on this object is the whole reason §8.2's trade is acceptable at all. +func TestEscrowGet_LeavesARecord(t *testing.T) { + h, st, _ := newTestHandler(t) + seedEscrowedHost(t, st, "h1", "c1", "HKEY", []byte("age-wrapped")) + + if rr := do(h, http.MethodGet, "/hosts/h1/escrow", "HKEY", ""); rr.Code != http.StatusOK { + t.Fatalf("GET = %d", rr.Code) + } + ev, err := st.GetLatestEventByType("c1", eventEscrowBlobServed) + if err != nil { + t.Fatal(err) + } + if ev == nil { + t.Fatal("R-199: a sealed recovery blob was served and NOTHING recorded it — the audit row is " + + "the mitigation that makes this endpoint's trade acceptable") + } + if ev.Severity != "warning" { + t.Errorf("severity = %q, want warning: info is an intentional non-notify, so the operator would "+ + "never hear that the recovery path was used", ev.Severity) + } + if !contains(ev.Message, "h1") { + t.Errorf("the record must name the host, got %q", ev.Message) + } + // The record must not carry the blob itself — it is an audit row, not a second copy. + if contains(ev.Message, "age-wrapped") || contains(ev.DetailsJSON, "age-wrapped") || + contains(ev.DetailsJSON, base64.StdEncoding.EncodeToString([]byte("age-wrapped"))) { + t.Errorf("the audit record embedded the blob: msg=%q details=%q", ev.Message, ev.DetailsJSON) + } + // A "no blob" answer is NOT a retrieval and must not raise one. + seedEscrowedHost(t, st, "h2", "c2", "KEY2", nil) + do(h, http.MethodGet, "/hosts/h2/escrow", "KEY2", "") + if ev, _ := st.GetLatestEventByType("c2", eventEscrowBlobServed); ev != nil { + t.Fatal("a present=false answer served no blob and must raise no retrieval record") + } +} + +// Operator-tier by construction, registered in the same commit that mints the type. +func TestEscrowBlobServed_IsOperatorOnly(t *testing.T) { + if !notify.IsOperatorOnly(eventEscrowBlobServed) { + t.Fatalf("%s is not registered operator-only — a customer would receive operator-grade English "+ + "about their sealed recovery bundle being handed out", eventEscrowBlobServed) + } +} + +// Scenario G — the operator-driven DR path is UNTOUCHED: same gate, same behaviour. The new sibling +// must not loosen it, and must not share it. +func TestEscrowGet_OperatorDRPathUnchanged(t *testing.T) { + h, st, _ := newTestHandler(t) + seedEscrowedHost(t, st, "h1", "c1", "HKEY", []byte("age-wrapped")) + + // re-enroll without recovery mode: still refused. + if rr := do(h, http.MethodPost, "/hosts/h1/re-enroll", "HKEY", `{"new_api_key":"x"}`); rr.Code != http.StatusForbidden { + t.Fatalf("re-enroll without recovery mode = %d, want 403 — the new endpoint must not loosen it", rr.Code) + } + // restore-directive without recovery mode: still refused. + if rr := do(h, http.MethodGet, "/hosts/h1/restore-directive", "HKEY", ""); rr.Code != http.StatusForbidden { + t.Fatalf("restore-directive without recovery mode = %d, want 403", rr.Code) + } + // …and arming it still works, i.e. the old path is functional, not merely closed. + if err := st.SetRecoveryMode("h1", time.Now().UTC().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + if rr := do(h, http.MethodGet, "/hosts/h1/restore-directive", "HKEY", ""); rr.Code != http.StatusOK { + t.Fatalf("restore-directive WITH recovery mode = %d, want 200", rr.Code) + } + // The new endpoint does NOT depend on recovery mode (that is the §8.2 trade, made explicit here so + // a future change to escrowSelfServiceRetrieval is visible as a test change). + if err := st.ClearRecoveryMode("h1"); err != nil { + t.Fatal(err) + } + if rr := do(h, http.MethodGet, "/hosts/h1/escrow", "HKEY", ""); rr.Code != http.StatusOK { + t.Fatalf("self-service escrow GET with recovery mode OFF = %d, want 200 (escrowSelfServiceRetrieval=true)", rr.Code) + } +} diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 92ae56c..d5ca696 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -239,6 +239,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow") h.handleHostEscrowPut(w, r, hostID) + // R-199 (v0.94.0): the box-authenticated MIRROR of the PUT above — a host reads back its own + // opaque identity blob so it can be unsealed with the customer's recovery code. Distinct from the + // operator-driven DR path in dr.go, which stays exactly as it is (see handleHostEscrowGet). + case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"): + hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow") + h.handleHostEscrowGet(w, r, hostID) // G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the // operator retrieves it via the /admin/ path (global key only). case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"): @@ -1198,6 +1204,125 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa // registered operator-only in notify.operatorOnlyEvents. const eventRepoKeyChanged = "offsite_repo_key_changed" +// eventEscrowBlobServed (R-199) — a host retrieved its own sealed identity blob. Hub-internal, +// operator-only. See handleHostEscrowGet for why every retrieval is loud. +const eventEscrowBlobServed = "escrow_blob_served" + +// escrowSelfServiceRetrieval is THE SINGLE DECISION POINT for the §8.2/§8.3 trade (R-199). +// +// true (§8.2, shipped v0.94.0) — a host may read its own blob whenever it authenticates as itself. +// false (§8.3, the fallback) — the same read additionally requires operator-armed recovery mode. +// +// It is one condition on purpose: the operator may overrule the trade below, and switching must cost a +// boolean rather than a redesign. Everything else in the recovery chain is identical either way. +const escrowSelfServiceRetrieval = true + +// handleHostEscrowGet serves a host its OWN opaque identity-escrow blob (R-199, v0.94.0). +// +// WHAT THIS GIVES OUT, WHY IT IS SAFE, AND WHAT IT CHANGES ABOUT WHO IS REQUIRED — recorded here so the +// next reader finds the trade rather than inferring it (the dr.go header convention). +// +// WHAT: the age-wrapped `IdentityBundle` — opaque ciphertext. It carries the offsite restic repository +// password, the tunnel token, the PBS token and the WG key. The hub stores these bytes and has no +// decrypt path; the recovery code R that opens them exists only in the customer's hands. +// +// WHY IT IS SAFE TO GIVE OUT: the blob is useless without R (age scrypt + ChaCha20-Poly1305; a wrong R +// fails closed at the KDF, never to a plausible-but-wrong bundle), and a 10-word EFF code carries ~129 +// bits. The caller already authenticates as this host for its report, its desired state, its WG +// registration and its PBS token — this adds no new identity, only a new object, and it is the exact +// MIRROR of the PUT above, which is how the blob got here in the first place. +// +// WHAT IT CHANGES, STATED PLAINLY BECAUSE IT IS THE WHOLE OF THE TRADE: before this, obtaining the blob +// required the OPERATOR to arm recovery mode with the global key (dr.go). Now whoever controls a +// rebuilt box can obtain it with that box's own credential. That is a real reduction in the number of +// parties required. They still cannot open it. The mitigation is that the capability is AUDITED rather +// than silent: every successful retrieval raises an operator event (below), because a silent capability +// is the shape this project has spent two weeks removing. +// +// THE OPERATOR-DRIVEN DR PATH IS UNTOUCHED. `handleReEnroll` / `handleGetRestoreDirective` keep their +// recovery-mode gate and their global-key arming, and they serve the K-escrow and the directive as +// well. This endpoint serves ONE object to ONE authenticated owner. Do not merge them. +func (h *Handler) handleHostEscrowGet(w http.ResponseWriter, r *http.Request, pathHostID string) { + authHostID, _, isGlobal, ok := h.checkAuthHost(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if pathHostID == "" { + http.Error(w, "Missing host_id", http.StatusBadRequest) + return + } + // SELF-SCOPED: a per-host key reads only its OWN escrow. The global operator key may read any — + // the same asymmetry the PUT has. Without this line any host key is a fleet-wide blob reader. + if !isGlobal && authHostID != pathHostID { + h.logger.Printf("[WARN] escrow GET REFUSED: host %s asked for %s's blob (self-scope)", authHostID, pathHostID) + http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden) + return + } + host, err := h.store.GetHost(pathHostID) + if err != nil { + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + if host == nil { + http.Error(w, "Unknown host_id", http.StatusNotFound) + return + } + // §8.3 fallback lives here and nowhere else. + if !escrowSelfServiceRetrieval && !host.InRecoveryMode(time.Now().UTC()) { + h.logger.Printf("[WARN] escrow GET REFUSED for %s — self-service retrieval is disabled and recovery mode is not armed", pathHostID) + http.Error(w, "Forbidden: host not in recovery mode (operator must arm it)", http.StatusForbidden) + return + } + + bundle, berr := h.store.GetHostDRBundle(pathHostID) + if berr != nil { + h.logger.Printf("[ERROR] escrow GET for %s: %v", pathHostID, berr) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + // A host with no sealed bundle gets a CLEAN ANSWER, not a fault: 200 with present=false. A 404 + // here would be indistinguishable from an unknown host, and an empty 200 without the flag would be + // indistinguishable from a zero-length blob — both read as "something is broken" to a caller whose + // situation is simply "no ceremony has run yet". + if bundle == nil || len(bundle.IdentityBlob) == 0 { + h.logger.Printf("[INFO] escrow GET for %s: no identity blob stored (no ceremony has run)", pathHostID) + writeJSON(w, http.StatusOK, map[string]any{"host_id": pathHostID, "present": false, "identity_escrow_b64": ""}) + return + } + + // THE MITIGATION (§8.2). Recorded BEFORE the bytes leave, so a retrieval cannot be served without + // its audit row; a save failure is logged and does NOT block the response (the blob is opaque and + // refusing it would break a recovery over an audit hiccup — but the log line always exists). + // + // SEVERITY = warning, i.e. it reaches the operator by e-mail. Retrieval is not routine today: it + // happens during a recovery and nowhere else. IF a customer-facing self-service flow ever makes it + // routine, revisit this — but revisit it deliberately, do not let it decay to info because the + // mail became annoying. + if host.CustomerID != "" { + msg := fmt.Sprintf("Recovery blob served: host %s retrieved its own sealed identity escrow (%d opaque bytes). "+ + "This is the recovery path in use — the blob cannot be opened without the customer's recovery code, which the hub never holds. "+ + "If no recovery is in progress on that box, investigate.", pathHostID, len(bundle.IdentityBlob)) + details, _ := json.Marshal(map[string]any{ + "host_id": pathHostID, + "blob_bytes": len(bundle.IdentityBlob), + "self_scope": !isGlobal, + }) + if _, eerr := h.store.SaveEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub"); eerr != nil { + h.logger.Printf("[WARN] %s event save FAILED for %s (serving anyway): %v", eventEscrowBlobServed, pathHostID, eerr) + } else if h.dispatcher != nil { + go h.dispatcher.ProcessEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub") + } + } + h.logger.Printf("[WARN] escrow blob SERVED to host %s (%d opaque bytes, self_scope=%v) — recovery path in use", + pathHostID, len(bundle.IdentityBlob), !isGlobal) + writeJSON(w, http.StatusOK, map[string]any{ + "host_id": pathHostID, + "present": true, + "identity_escrow_b64": base64.StdEncoding.EncodeToString(bundle.IdentityBlob), + }) +} + // maybeEmitRepoKeyChanged raises ONE operator signal per supersession when the sealed offsite repo // password demonstrably changed. Both hashes have been stored since SLICE 3 (host_escrow and, since // v0.60.0, host_escrow_superseded) and NOTHING compared them: demo-felhom's repository password diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 859c0f1..17501c9 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -438,6 +438,12 @@ var operatorOnlyEvents = map[string]bool{ // only: the operator channel is untouched. "offsite_delivery_stuck": true, "offsite_credential_restaged": true, + // R-199 (v0.94.0). A host retrieved its own sealed recovery blob. Operator-tier by construction: + // it names host ids and opaque byte counts, the customer can take no action on it, and its whole + // purpose is that the operator sees a capability being used. Registered in the same commit that + // mints the type — an operator-tier type absent from this register reaches customers as raw + // English (the v0.78.0 defect recorded above). + "escrow_blob_served": true, } // IsOperatorOnly reports whether an event type is barred from customer dispatch. Exported so the