hub v0.60.0: offsite continuity Part B — superseded-escrow retention (data-first)

- host_escrow_superseded table + SaveHostEscrow retains a different-sha old blob before overwrite (tx); same-sha idempotent (no supersede row); returns superseded bool. ACK/restore read the current row unchanged. CountSuperseded/ListSuperseded; DeleteHost drops retained rows.
- escrow_superseded audit event + operator retained-count on host detail; register offbox_repo_orphaned/reset. Red-proof TestSaveHostEscrow_RetainsSuperseded.
This commit is contained in:
2026-07-17 10:47:50 +02:00
parent 1c737db4f4
commit e247dbc1be
13 changed files with 217 additions and 18 deletions
+24
View File
@@ -1,5 +1,29 @@
# Felhom Hub — Changelog
## v0.60.0 — offsite continuity Part B: superseded-escrow retention (data-first) (2026-07-17)
Closes the data-loss half of the reinstall-orphaned-repo incident: `SaveHostEscrow`'s destructive
`ON CONFLICT` overwrite meant a new escrow blob DESTROYED the old passphrase's only copy — so 18
snapshots keyed under the old password became unrecoverable. Viktor's ruling (data protection first):
**retain superseded blobs** so the old passphrase stays customer-R-recoverable. Pairs with controller
v0.142.0 (Part A orphaned-repo guard). Green: `go build ./... && go vet ./... && go test ./...`.
- **`host_escrow_superseded` (new table) + `SaveHostEscrow` rewrite.** On upload, if the current row
seals a DIFFERENT `restic_pw_sha256`, the old row is COPIED into the history table (in one tx)
BEFORE the current row is overwritten; a same-sha re-upload (idempotent re-ceremony) refreshes the
current row and creates NO supersede row. `SaveHostEscrow` now returns `superseded bool`. Retain
ALL (no pruning — the blobs are tiny + R-encrypted, custody unchanged); the hub still never
decrypts. **ACK/restore-serving read the CURRENT row (`GetHostEscrow`) — unchanged.** New
`CountSupersededEscrow` / `ListSupersededEscrow` (the latter seeds the future R-26 recovery flow).
`DeleteHost(deleteEscrow=true)` also drops the retained rows.
- **Surfaces.** Upload handler emits the hub-internal `escrow_superseded` audit event + logs the
retained count; the operator host-detail DR/Backup panel shows "N superseded escrow blob(s)
retained". Registered `offbox_repo_orphaned` / `offbox_repo_reset` (controller v0.142.0 pushes) in
`allowedEventTypes` + `customerMessages`.
- Red-proof `TestSaveHostEscrow_RetainsSuperseded` (pre-fix destructive overwrite → old blob gone →
FAIL; fixed → retained + retrievable; same-sha idempotent).
- **Deploy:** bump `manifests/hub.yaml` image tag to `0.60.0` and sync.
## v0.59.0 — Direction-2a: agent-plane immediate-sync poke sender + ep0 felhom-poke surface (2026-07-16)
Implements the AGENT-plane half of `documentation/audits/SPIKE-immediate-sync-transport-2026-07-16.md`
+1 -1
View File
@@ -155,7 +155,7 @@ func TestReportACK_EscrowStatus(t *testing.T) {
}
// escrow row with identity blob + hash → the ACK carries all three fields
if err := st.SaveHostEscrow("hv1", []byte("k-blob"), "fp", "zero_knowledge", "2026-07-09T20:00:00Z", "abc123"); err != nil {
if _, err := st.SaveHostEscrow("hv1", []byte("k-blob"), "fp", "zero_knowledge", "2026-07-09T20:00:00Z", "abc123"); err != nil {
t.Fatal(err)
}
if err := st.SaveHostDRBundle("hv1", []byte("identity-blob"), "{}"); err != nil {
+22 -3
View File
@@ -1095,12 +1095,27 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa
if createdAt == "" {
createdAt = time.Now().UTC().Format(time.RFC3339)
}
// Store the OPAQUE bytes. No decrypt path exists — the hub cannot open this.
if err := h.store.SaveHostEscrow(pathHostID, blob, req.KeyFingerprint, req.Posture, createdAt, req.ResticPwSHA256); err != nil {
h.logger.Printf("[ERROR] Failed to store escrow for host %s: %v", pathHostID, err)
// Store the OPAQUE bytes. No decrypt path exists — the hub cannot open this. Part B (v0.60.0):
// when this upload supersedes a DIFFERENT-passphrase old blob, the old one is RETAINED (not
// overwritten) so its recovery-code-recoverable history survives (Viktor's data-first ruling).
superseded, serr := h.store.SaveHostEscrow(pathHostID, blob, req.KeyFingerprint, req.Posture, createdAt, req.ResticPwSHA256)
if serr != nil {
h.logger.Printf("[ERROR] Failed to store escrow for host %s: %v", pathHostID, serr)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
if superseded {
n, _ := h.store.CountSupersededEscrow(pathHostID)
h.logger.Printf("[INFO] escrow for host %s superseded a different-passphrase blob — RETAINED (now %d superseded blob(s) held)", pathHostID, n)
// Hub-internal audit event (not gated by allowedEventTypes) — tied to the owning customer.
if host, herr := h.store.GetHost(pathHostID); herr == nil && host != nil && host.CustomerID != "" {
details, _ := json.Marshal(map[string]any{"host_id": pathHostID, "retained_count": n})
if _, eerr := h.store.SaveEvent(host.CustomerID, "escrow_superseded", "info",
"A korábbi helyreállítási csomag megőrizve (új kulcs érkezett).", string(details), "hub"); eerr != nil {
h.logger.Printf("[WARN] escrow_superseded event save failed for %s: %v", pathHostID, eerr)
}
}
}
// Slice 10D.1: optionally store the IDENTITY escrow blob + the non-secret DR directive alongside
// the K-escrow (both opaque / non-secret — no usable secret hub-side). Additive: a slice-7
// upload without these is unchanged.
@@ -1503,6 +1518,10 @@ var allowedEventTypes = map[string]bool{
// dynamic Hungarian message is customer-grade — deliberately NO customerMessages entry, which would
// discard the numbers (templates.go:129 priority)).
"offbox_enlarge_blocked": true,
// controller v0.142.0 — offsite-repo continuity: the remote repo is orphaned (reinstall shape) /
// was reset (move-aside + re-init). Customer-grade messages below.
"offbox_repo_orphaned": true,
"offbox_repo_reset": true,
"storage_disconnected": true,
"storage_reconnected": true,
"disk_warning": true,
+3
View File
@@ -62,6 +62,9 @@ var customerMessages = map[string]string{
"backup_integrity_failed": "A mentés integritás ellenőrzés hibát talált!",
"crossdrive_completed": "A másodlagos mentés sikeresen elkészült.",
"crossdrive_failed": "A másodlagos mentés sikertelen!",
// Offsite-repo continuity (controller v0.142.0)
"offbox_repo_orphaned": "A távoli mentési tároló elárvult: a benne lévő mentések egy korábbi, már nem elérhető kulccsal készültek (jellemzően újratelepítés után). Új mentés a tároló visszaállításáig nem készül — nyisd meg a Távoli mentés oldalt.",
"offbox_repo_reset": "A távoli mentési tároló visszaállítva: a régi előzmény félretéve (nem törölve), és egy üres, új tároló jött létre a mostani kulccsal.",
// Disk events (GUEST — the controller's own cgroup view)
"disk_warning": "A lemezterület 90% felett van — kérjük, szabadíts fel helyet.",
+2 -2
View File
@@ -226,7 +226,7 @@ func TestReissue_InvalidatesEscrow(t *testing.T) {
if err := st.UpsertHost(&store.Host{HostID: cust + "-01", CustomerID: cust, APIKey: "k"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob"), "SHA256:fp", "zero_knowledge", "2026-07-16T00:00:00Z", "OLDHASH"); err != nil {
if _, err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob"), "SHA256:fp", "zero_knowledge", "2026-07-16T00:00:00Z", "OLDHASH"); err != nil {
t.Fatal(err)
}
// Before re-issue: current escrow — the hub serves the sealed hash and is NOT stale.
@@ -256,7 +256,7 @@ func TestReissue_InvalidatesEscrow(t *testing.T) {
}
// A fresh ceremony (new blob sealing the new password) clears stale + serves the new hash.
if err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob-2"), "SHA256:fp", "zero_knowledge", "2026-07-16T01:00:00Z", "NEWHASH"); err != nil {
if _, err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob-2"), "SHA256:fp", "zero_knowledge", "2026-07-16T01:00:00Z", "NEWHASH"); err != nil {
t.Fatal(err)
}
es, _ = st.GetEscrowStatusForCustomer(cust)
@@ -0,0 +1,58 @@
package store
import "testing"
// Part B (v0.60.0) — the escrow retention red-proof (the 2026-07-17 incident class). A new escrow blob
// with a DIFFERENT sealed-passphrase sha must RETAIN the old blob (recoverable) instead of destroying
// it; a same-sha re-upload is idempotent (no supersede row). Pre-fix (destructive ON CONFLICT
// overwrite) the old blob is gone → the retrieval assertion FAILS.
func TestSaveHostEscrow_RetainsSuperseded(t *testing.T) {
st := newTestStore(t)
const h = "h1"
// 1st upload (P_old) — nothing to supersede.
sup, err := st.SaveHostEscrow(h, []byte("blob-old"), "fp-old", "zk", "2026-07-09T00:00:00Z", "SHA_OLD")
if err != nil {
t.Fatal(err)
}
if sup {
t.Fatal("first upload must not supersede")
}
// 2nd upload (P_new, DIFFERENT sha) — must supersede + retain the old.
sup, err = st.SaveHostEscrow(h, []byte("blob-new"), "fp-new", "zk", "2026-07-16T00:00:00Z", "SHA_NEW")
if err != nil {
t.Fatal(err)
}
if !sup {
t.Fatal("a different-passphrase upload must supersede (retain the old blob)")
}
// Current row is the NEW blob (ACK/restore-serving read this — unchanged behavior).
cur, _ := st.GetHostEscrow(h)
if cur == nil || string(cur.Blob) != "blob-new" || cur.ResticPwSHA256 != "SHA_NEW" {
t.Fatalf("current row not the new blob: %+v", cur)
}
// RED-PROOF: the OLD blob is RETAINED and retrievable.
old, err := st.ListSupersededEscrow(h)
if err != nil {
t.Fatal(err)
}
if len(old) != 1 || string(old[0].Blob) != "blob-old" || old[0].ResticPwSHA256 != "SHA_OLD" {
t.Fatalf("old blob NOT retained after overwrite (incident): %+v", old)
}
if n, _ := st.CountSupersededEscrow(h); n != 1 {
t.Fatalf("superseded count = %d, want 1", n)
}
// 3rd upload, SAME sha as current — idempotent (re-ceremony of the same password): NO supersede row.
sup, err = st.SaveHostEscrow(h, []byte("blob-new-2"), "fp-new", "zk", "2026-07-16T01:00:00Z", "SHA_NEW")
if err != nil {
t.Fatal(err)
}
if sup {
t.Fatal("a same-sha re-upload must NOT supersede (idempotent)")
}
if n, _ := st.CountSupersededEscrow(h); n != 1 {
t.Fatalf("idempotent re-upload created a superseded row: count=%d", n)
}
}
+1 -1
View File
@@ -57,7 +57,7 @@ func seedHostWithArtifacts(t *testing.T, s *Store, hostID, customerID string) {
"PK-"+hostID, "ip-"+hostID, hostID); err != nil {
t.Fatal(err)
}
if err := s.SaveHostEscrow(hostID, []byte("opaque-escrow"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
if _, err := s.SaveHostEscrow(hostID, []byte("opaque-escrow"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
}
+97 -5
View File
@@ -330,6 +330,25 @@ func (s *Store) migrate() error {
updated_at DATETIME NOT NULL DEFAULT (datetime('now'))
);
-- host_escrow_superseded (v0.60.0, offsite-continuity): RETAINED old escrow blobs. Viktor's
-- ruling (data protection first): when a NEW escrow blob supersedes an old one whose sealed
-- restic-password sha DIFFERS, the old row is COPIED here BEFORE host_escrow is overwritten
-- so the old passphrase stays customer-R-recoverable (turns the reinstall-orphan incident from
-- "history destroyed" into "history recoverable with the recovery code"). Append-only; the hub
-- never decrypts; NO pruning (the blobs are tiny + R-encrypted; custody unchanged). The ACK and
-- restore-serving read host_escrow (the CURRENT row) never this table.
CREATE TABLE IF NOT EXISTS host_escrow_superseded (
id INTEGER PRIMARY KEY AUTOINCREMENT,
host_id TEXT NOT NULL,
blob BLOB NOT NULL,
key_fingerprint TEXT NOT NULL DEFAULT '',
posture TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL,
restic_pw_sha256 TEXT NOT NULL DEFAULT '',
superseded_at DATETIME NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_host_escrow_superseded_host ON host_escrow_superseded(host_id);
-- signed_jobs (slice 10A): the per-host queue of OPAQUE operator-signed destructive-op
-- blobs. The hub STORES + SERVES them; it never forges one (there is no signing key
-- hub-side) and never executes them (execution + signature verification is slice 10B).
@@ -1990,6 +2009,7 @@ func (s *Store) DeleteHost(hostID string, deleteEscrow bool) error {
}
if deleteEscrow {
stmts = append(stmts, `DELETE FROM host_escrow WHERE host_id = ?`)
stmts = append(stmts, `DELETE FROM host_escrow_superseded WHERE host_id = ?`) // retained blobs go with the host
}
stmts = append(stmts, `DELETE FROM hosts WHERE host_id = ?`)
for _, q := range stmts {
@@ -2071,8 +2091,49 @@ type HostEscrow struct {
// SaveHostEscrow stores (last-write-wins) the OPAQUE escrow blob for a host. The hub keeps the
// bytes and NEVER decrypts them — there is no decrypt path. createdAt is the agent's timestamp.
// resticPwSHA256 is "" when the ceremony sealed no staged password (stored as-is; never auto-confirms).
func (s *Store) SaveHostEscrow(hostID string, blob []byte, keyFingerprint, posture, createdAt, resticPwSHA256 string) error {
_, err := s.db.Exec(`
// SaveHostEscrow returns superseded=true when it RETAINED a different-passphrase old blob into
// host_escrow_superseded before overwriting the current row (Part B, v0.60.0). A same-sha re-upload
// (idempotent re-ceremony of the same password) refreshes the current row and does NOT create a
// superseded row.
func (s *Store) SaveHostEscrow(hostID string, blob []byte, keyFingerprint, posture, createdAt, resticPwSHA256 string) (superseded bool, err error) {
tx, err := s.db.Begin()
if err != nil {
return false, err
}
defer func() {
if err != nil {
tx.Rollback()
}
}()
// Retain the current row iff it exists AND seals a DIFFERENT restic password (the incident: a
// recreated volume mints a new passphrase; the old must stay recoverable with its recovery code).
var (
curBlob []byte
curFp, curPosture, curCreated, curSHA string
exists bool
)
row := tx.QueryRow(`SELECT blob, key_fingerprint, posture, created_at, COALESCE(restic_pw_sha256,'') FROM host_escrow WHERE host_id = ?`, hostID)
switch scanErr := row.Scan(&curBlob, &curFp, &curPosture, &curCreated, &curSHA); scanErr {
case nil:
exists = true
case sql.ErrNoRows:
exists = false
default:
err = scanErr
return false, err
}
if exists && curSHA != resticPwSHA256 {
if _, err = tx.Exec(`
INSERT INTO host_escrow_superseded (host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, superseded_at)
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))`,
hostID, curBlob, curFp, curPosture, curCreated, curSHA); err != nil {
return false, err
}
superseded = true
}
if _, err = tx.Exec(`
INSERT INTO host_escrow (host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, updated_at)
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
ON CONFLICT(host_id) DO UPDATE SET
@@ -2083,9 +2144,40 @@ func (s *Store) SaveHostEscrow(hostID string, blob []byte, keyFingerprint, postu
restic_pw_sha256 = excluded.restic_pw_sha256,
stale_at = NULL,
updated_at = datetime('now')`,
hostID, blob, keyFingerprint, posture, createdAt, resticPwSHA256,
)
return err
hostID, blob, keyFingerprint, posture, createdAt, resticPwSHA256); err != nil {
return false, err
}
err = tx.Commit()
return superseded, err
}
// CountSupersededEscrow returns how many retained (superseded) escrow blobs the hub holds for a host
// (the operator "N superseded escrow blobs retained" surface). 0 on no rows.
func (s *Store) CountSupersededEscrow(hostID string) (int, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM host_escrow_superseded WHERE host_id = ?`, hostID).Scan(&n)
return n, err
}
// ListSupersededEscrow returns the retained (superseded) escrow blobs for a host, newest-superseded
// first. Opaque bytes — the hub never decrypts. Seeds the future guided-recovery flow (R-26).
func (s *Store) ListSupersededEscrow(hostID string) ([]HostEscrow, error) {
rows, err := s.db.Query(`
SELECT host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, superseded_at
FROM host_escrow_superseded WHERE host_id = ? ORDER BY id DESC`, hostID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []HostEscrow
for rows.Next() {
var e HostEscrow
if err := rows.Scan(&e.HostID, &e.Blob, &e.KeyFingerprint, &e.Posture, &e.CreatedAt, &e.ResticPwSHA256, &e.UpdatedAt); err != nil {
return nil, err
}
out = append(out, e)
}
return out, rows.Err()
}
// MarkEscrowStale flags a host's escrow blob as stale (v0.57.0, 2.3) — called when the offsite repo
+3
View File
@@ -406,6 +406,9 @@ func (s *Server) hostDetailData(host *store.Host, r *http.Request) map[string]in
"NeedsDRMigration": capabilitiesNeedDRMigration(capabilities),
"DRPresent": drBundle != nil,
"EscrowPresent": escrow != nil,
// v0.60.0 Part B: retained superseded escrow blobs (data-first — old passphrases stay
// R-recoverable). Operator-only surface.
"SupersededEscrowCount": func() int { n, _ := s.store.CountSupersededEscrow(host.HostID); return n }(),
// v0.46.0 Diagnostics: pending log pulls + received/blocked bundles (72 h TTL).
"LogBundles": s.hostLogBundleRows(host),
"CSRFToken": s.getCSRFToken(r),
+3 -3
View File
@@ -64,7 +64,7 @@ func TestHostDelete_EscrowAckRequired(t *testing.T) {
if err := st.UpsertHost(&store.Host{HostID: "esc-host", CustomerID: "c2", APIKey: "k"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostEscrow("esc-host", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
if _, err := st.SaveHostEscrow("esc-host", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
@@ -110,7 +110,7 @@ func TestHostDelete_ImpactJSON(t *testing.T) {
if err := st.UpsertHost(&store.Host{HostID: "imp-host", CustomerID: "c4", APIKey: "SECRET-KEY"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostEscrow("imp-host", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
if _, err := st.SaveHostEscrow("imp-host", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
if err := st.UpsertGuestFromReport(&store.Guest{GuestID: store.GuestID("imp-host", 100),
@@ -159,7 +159,7 @@ func TestHostDelete_HappyPath(t *testing.T) {
if err := st.UpsertHost(&store.Host{HostID: "dr-drill", CustomerID: "c5", APIKey: "k"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostEscrow("dr-drill", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
if _, err := st.SaveHostEscrow("dr-drill", []byte("blob"), "fp", "p", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
rr := postHostDelete(t, s, "dr-drill", url.Values{
+1 -1
View File
@@ -130,7 +130,7 @@ func TestHandleHostDetail(t *testing.T) {
t.Fatal(err)
}
// DR + escrow present (escrow row must exist before the DR bundle UPDATE).
if err := st.SaveHostEscrow("demo-felhom-01", []byte("opaque-escrow"), "fp", "posture", "2026-06-01T00:00:00Z", ""); err != nil {
if _, err := st.SaveHostEscrow("demo-felhom-01", []byte("opaque-escrow"), "fp", "posture", "2026-06-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
if err := st.SaveHostDRBundle("demo-felhom-01", []byte("opaque-identity"), `{"v":1}`); err != nil {
+1 -1
View File
@@ -390,7 +390,7 @@ func TestPBSDR_F14AutoReissueOnAckedDeletion(t *testing.T) {
if err := st.UpsertHost(&store.Host{HostID: "peti-00-dead", CustomerID: "peti", APIKey: "oldkey"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostEscrow("peti-00-dead", []byte("opaque"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
if _, err := st.SaveHostEscrow("peti-00-dead", []byte("opaque"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
if err := st.DeleteHost("peti-00-dead", true); err != nil {
@@ -263,7 +263,7 @@
</div>
<div class="info-item">
<span class="label">Key Escrow</span>
<span class="value">{{if .EscrowPresent}}<span style="color: var(--green)">present</span>{{else}}<span class="text-muted">none</span>{{end}}</span>
<span class="value">{{if .EscrowPresent}}<span style="color: var(--green)">present</span>{{else}}<span class="text-muted">none</span>{{end}}{{if gt .SupersededEscrowCount 0}} <span class="text-muted">· {{.SupersededEscrowCount}} superseded escrow blob(s) retained</span>{{end}}</span>
</div>
</div>
</section>