From 1db56bf8379df68ee23e7b000c5ff01bd76b5c79 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 12 Aug 2026 18:40:00 +0200 Subject: [PATCH] =?UTF-8?q?v0.129.0=20=E2=80=94=20a=20correct=20code=20for?= =?UTF-8?q?=20an=20earlier=20package=20stops=20being=20called=20wrong=20(R?= =?UTF-8?q?-311)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yesterday's drill proved a retained escrow package opens a set-aside store and restores planted files byte-identical, while this agent answered the customer's correct code with "the recovery code did not open the sealed bundle". Nothing had ever tried the retained packages, so a correct-but-earlier code and a mistype were genuinely indistinguishable. OffsiteKeyRecoverer gains an optional FetchRetained, consulted ONLY after the current package refuses, so the ordinary recovery pays nothing for it and cannot fail because of it. A match returns ErrCodeOpensRetained wrapped in a RetainedOpenedError carrying the supersession date - no material, no code, no password. The local API answers 422: a FIFTH status added to the R-224 switch, never a restructuring of it. Fail-safe in every direction. Nil fetcher, a hub too old for the route (404 is a clean "none"), a transport failure, a malformed package: each leaves the original refusal standing. Attempts bounded at 6 because each unwrap is ~1s of scrypt. Seven tests with REAL age crypto - the two situations are indistinguishable AT THE UNWRAP, so a faked unwrap would prove nothing. Red-proof asserted applied: remove the retained lookup and the fail-closed wrong-code error returns, which is the lie in those exact words. --- CHANGELOG.md | 34 ++++ cmd/felhom-agent/main.go | 53 ++++-- internal/escrow/recover.go | 117 +++++++++++- internal/escrow/recover_retained_test.go | 230 +++++++++++++++++++++++ internal/hub/client.go | 66 +++++++ internal/localapi/escrow_recover.go | 29 +++ 6 files changed, 514 insertions(+), 15 deletions(-) create mode 100644 internal/escrow/recover_retained_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1a30b..8511619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,37 @@ +## v0.129.0 — a correct code for an earlier package stops being called wrong (2026-08-12, R-311) + +**The measurement this fixes.** On 2026-08-12 a recovery code that provably opens a RETAINED package +— unsealed by hand, and it restored planted files byte-identical from a store the box itself could no +longer open — was answered by this agent with *"the recovery code did not open the sealed bundle"*. +The code was correct. Nothing had ever tried the retained packages, so the engine could not tell a +correct-but-earlier code from a mistype, and the screen said so out loud: a true sentence about our +own incuriosity, read by the customer as a statement about their code. + +**`OffsiteKeyRecoverer` gains an optional `FetchRetained`.** It is consulted ONLY after the current +package has refused, so the ordinary recovery pays nothing for it and cannot fail because of it. When +one of the retained packages opens, the recoverer returns `ErrCodeOpensRetained` wrapped in a +`RetainedOpenedError` carrying the supersession date — no material, no code, no password. + +**The local API answers 422** ("your code is correct, it belongs to an EARLIER sealed package") — a +FIFTH status added to the R-224 switch, not a restructuring of it. 422 rather than 400 because the +request was well-formed AND the credential valid; a 400 would put it in the same bucket as a mistype, +which is the defect. + +**Fail-safe in every direction.** A nil fetcher, a hub too old to have the route (404 is a clean +"none"), a transport failure, a malformed package: each leaves the original refusal standing, +unchanged. The worst outcome of this feature breaking is the behaviour we had before it existed. +Attempts are bounded (`MaxRetainedTried`, default 6) because each unwrap is ~1 s of scrypt by design +and an unbounded loop would turn one wrong code into a minutes-long hang. + +**New hub client call:** `FetchRetainedIdentityEscrow` → `GET /api/v1/hosts//escrow/retained` +(hub >= v0.103.0), self-scoped by the same per-host key. + +Seven tests with REAL age crypto, because the two situations are indistinguishable AT THE UNWRAP and a +faked unwrap would prove nothing about what was broken. Red-proof, asserted applied: removing the +retained lookup returns the fail-closed wrong-code error — **the lie comes back, in those words.** + +--- + ### Gates only — 2026-08-09 (no release, no version bump, no binary published) **Two guards, both owed since the 2026-08-09 install outage (R-273/R-287). Nothing that runs on a diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index dbcdd1a..605f5d0 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -1769,23 +1769,48 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St } return blob, true, nil }, + // R-311 — the RETAINED packages, wired here and ONLY here, on the same self-scoped hub client. + // Consulted only after the current package has refused the code (see tryRetained), so the + // ordinary recovery pays nothing for it and cannot fail because of it. + FetchRetained: func(ctx context.Context) ([]escrow.RetainedBlob, int, error) { + resp, ferr := hubClient.FetchRetainedIdentityEscrow(ctx) + if ferr != nil { + return nil, 0, ferr + } + out := make([]escrow.RetainedBlob, 0, len(resp.Packages)) + for _, p := range resp.Packages { + blob, derr := base64.StdEncoding.DecodeString(p.IdentityEscrowB64) + if derr != nil || len(blob) == 0 { + // One malformed package must not sink the rest — the customer's code may open a + // later one, and a skipped entry is strictly better than a refusal we cannot justify. + continue + } + out = append(out, escrow.RetainedBlob{ + Blob: blob, + SupersededAt: p.SupersededAt, + KeyFingerprint: p.KeyFingerprint, + Index: p.Index, + }) + } + return out, resp.UnopenableCount, nil + }, } srv, err := localapi.NewServer(localapi.Options{ EscrowRecovery: escrowRecoverer, - ListenAddr: cfg.LocalAPI.ListenAddr, - Cert: cert, - AgentVersion: version, // v0.82.0: the X-Felhom-Agent-Version capability channel - Guests: px, - Backups: runner, - BackupTiers: apiTiers, // R-82: primary first; untargeted endpoints act on the primary - InFlight: inFlight, // R-85: shared with the restore-test scheduler (Scenario F) - Store: store, - Storage: observer, - DriveTargets: driveTargets, // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages) - Smart: storage.NewSmartReader(hostOps), // v0.95.0 Fix B: SMART for the union-path drives - HostReader: storage.NewProcHostReader(), // Impl-2b: durableIDForMount raw-mount fallback + role gate - Tokens: tokens, - BackupCadence: cfg.Backup.BackupCadence(), + ListenAddr: cfg.LocalAPI.ListenAddr, + Cert: cert, + AgentVersion: version, // v0.82.0: the X-Felhom-Agent-Version capability channel + Guests: px, + Backups: runner, + BackupTiers: apiTiers, // R-82: primary first; untargeted endpoints act on the primary + InFlight: inFlight, // R-85: shared with the restore-test scheduler (Scenario F) + Store: store, + Storage: observer, + DriveTargets: driveTargets, // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages) + Smart: storage.NewSmartReader(hostOps), // v0.95.0 Fix B: SMART for the union-path drives + HostReader: storage.NewProcHostReader(), // Impl-2b: durableIDForMount raw-mount fallback + role gate + Tokens: tokens, + BackupCadence: cfg.Backup.BackupCadence(), // Disk management (slice 8C): the privileged host surface + the data-bearing wipe gate. Disks: hostOps, DiskGate: storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID}, diff --git a/internal/escrow/recover.go b/internal/escrow/recover.go index c254773..a259e6a 100644 --- a/internal/escrow/recover.go +++ b/internal/escrow/recover.go @@ -47,17 +47,80 @@ var ( // retro-fitted, because R is never retained. Distinguished from a wrong code so the operator is // not sent hunting for a mistyped recovery code that was typed correctly. ErrNoResticPassword = errors.New("escrow: the recovered bundle carries NO offsite repository password (a pre-fork-4 blob — the field did not exist when it was sealed and cannot be retro-fitted)") + // ErrCodeOpensRetained — the code did NOT open the package the hub currently holds, and DID open a + // RETAINED (earlier) one. R-311. + // + // ⚠ THIS IS NOT A FAILURE OF THE CUSTOMER'S. It is the single most important distinction on this + // path, because until 2026-08-12 it was indistinguishable from a mistype and was reported as one. + // The screen could only say "it may be a typo, or it may be an older code, and we cannot tell them + // apart from here" — and it could not tell them apart because NOTHING EVER LOOKED. Now something + // looks, so the sentence can stop hedging. + // + // It carries no material and no code: only WHICH earlier package opened, by its supersession date, + // which is the one fact the customer needs to recognise it. + ErrCodeOpensRetained = errors.New("escrow: the recovery code did not open the CURRENT sealed package, but it DID open a retained earlier one") ) +// RetainedMatch says which retained package a code opened. Returned inside RetainedOpenedError; it +// carries no secret — not the code, not the bundle, not the repository password. +type RetainedMatch struct { + // SupersededAt is when this package stopped being the current one (hub-supplied, RFC3339-ish). + // It is what the recovery screen shows so the customer can recognise which code they are holding. + SupersededAt string + // KeyFingerprint is the escrow key fingerprint of that package — operator-log material only. + KeyFingerprint string + // Index is the hub's position label within ONE response. Not durable; do not persist it. + Index int + // HasResticPassword is false when the retained package opened but carries no repository password + // (a pre-fork-4 seal). The code is still CORRECT; the history behind it still cannot be reopened. + // Collapsing this into "recoverable" would repeat R-202's mistake on a new surface. + HasResticPassword bool +} + +// RetainedOpenedError wraps ErrCodeOpensRetained with the match. Callers classify with errors.Is on +// the sentinel and read the detail with errors.As. +type RetainedOpenedError struct { + Match RetainedMatch +} + +func (e *RetainedOpenedError) Error() string { + return ErrCodeOpensRetained.Error() + " (superseded_at=" + e.Match.SupersededAt + ")" +} +func (e *RetainedOpenedError) Unwrap() error { return ErrCodeOpensRetained } + // BlobFetcher yields this host's own opaque identity-escrow blob. present=false is a clean "none". // An interface-free func field keeps this package free of any dependency on the hub client. type BlobFetcher func(ctx context.Context) (blob []byte, present bool, err error) +// RetainedBlob is one retained sealed package as the recoverer sees it: opaque bytes plus the labels +// needed to name it. No secret. +type RetainedBlob struct { + Blob []byte + SupersededAt string + KeyFingerprint string + Index int +} + +// RetainedFetcher yields this host's RETAINED sealed packages, newest-superseded first. An empty +// slice is a clean "none". R-311. +type RetainedFetcher func(ctx context.Context) (blobs []RetainedBlob, unopenable int, err error) + // OffsiteKeyRecoverer is the assembled links 6→8. Construct it with a fetcher; call it with R. type OffsiteKeyRecoverer struct { Fetch BlobFetcher + // FetchRetained is OPTIONAL and consulted ONLY after the current package has refused the code. + // nil keeps the pre-R-311 behaviour exactly: a refusal stays a refusal. That is deliberate — an + // agent wired without it must not behave differently from one that has no retained packages. + FetchRetained RetainedFetcher + // MaxRetainedTried bounds the scrypt work a single wrong code can cost. Each attempt is ~1 s of + // KDF by design, so an unbounded loop over a long supersession history would turn one wrong code + // into a minutes-long hang on the customer's screen. 0 means the built-in default. + MaxRetainedTried int } +// defaultMaxRetainedTried — six attempts is ~6 s worst case, which is a slow screen and not a hang. +const defaultMaxRetainedTried = 6 + // RecoverOffsiteRepoPassword fetches, unseals and extracts. It returns ONLY the repository password. // // A WRONG RECOVERY CODE FAILS CLOSED at the scrypt KDF inside UnwrapIdentity — `age -d` exits @@ -86,10 +149,62 @@ func (r OffsiteKeyRecoverer) RecoverOffsiteRepoPassword(ctx context.Context, rec } bundle, err := UnwrapIdentityBundle(ctx, blob, recoveryCode) if err != nil { - return "", err // already the fail-closed "the recovery code did not unwrap…" message; no secret in it + // R-311 — BEFORE calling this a wrong code, ask whether it is the RIGHT code for an EARLIER + // package. The engine fails closed identically either way, so the two are indistinguishable + // from the unwrap alone; the only way to tell is to try. Until this existed nobody tried, and + // the screen said so out loud ("innen nem tudjuk megkülönböztetni őket") — a true sentence + // about our own incuriosity, read by the customer as a statement about their code. + if m, ok := r.tryRetained(ctx, recoveryCode); ok { + return "", &RetainedOpenedError{Match: m} + } + return "", err // the fail-closed "the recovery code did not unwrap…" message; no secret in it } if bundle.ResticRepoPassword == "" { return "", ErrNoResticPassword } return bundle.ResticRepoPassword, nil } + +// tryRetained reports whether the code opens one of this host's RETAINED packages, and which. +// +// FAILURE HERE IS SILENT AND MEANS "NO", NEVER "YES" and never a different verdict for the caller. A +// hub that cannot answer, a route an older hub does not have, a malformed blob — each leaves the +// original refusal standing, unchanged. That is the fail-safe direction: the worst outcome of this +// function breaking is the behaviour we had before it existed. +// +// NOTHING IS LOGGED HERE and no return value carries the code, a bundle or a password. +func (r OffsiteKeyRecoverer) tryRetained(ctx context.Context, recoveryCode string) (RetainedMatch, bool) { + if r.FetchRetained == nil { + return RetainedMatch{}, false + } + blobs, _, err := r.FetchRetained(ctx) + if err != nil || len(blobs) == 0 { + return RetainedMatch{}, false + } + limit := r.MaxRetainedTried + if limit <= 0 { + limit = defaultMaxRetainedTried + } + for i, rb := range blobs { + if i >= limit { + break + } + if len(rb.Blob) == 0 { + continue + } + bundle, uerr := UnwrapIdentityBundle(ctx, rb.Blob, recoveryCode) + if uerr != nil { + continue // this one is not the customer's; try the next + } + return RetainedMatch{ + SupersededAt: rb.SupersededAt, + KeyFingerprint: rb.KeyFingerprint, + Index: rb.Index, + // A retained package can itself predate the repository-password field. The code is still + // correct and must be told so — but the history behind it still cannot be reopened, and + // saying otherwise would be a promise this path cannot keep. + HasResticPassword: bundle.ResticRepoPassword != "", + }, true + } + return RetainedMatch{}, false +} diff --git a/internal/escrow/recover_retained_test.go b/internal/escrow/recover_retained_test.go new file mode 100644 index 0000000..07fb36a --- /dev/null +++ b/internal/escrow/recover_retained_test.go @@ -0,0 +1,230 @@ +package escrow + +import ( + "context" + "errors" + "fmt" + "testing" +) + +// R-311 — a correct code for an EARLIER package must stop being reported as a wrong code. +// +// These use REAL age crypto, like the R-199 tests beside them, because the whole point is that the +// two situations are indistinguishable AT THE UNWRAP: both fail closed on the current package. A +// faked unwrap would prove nothing about the thing that was actually broken. + +const testR2 = "another correct horse battery staple sedative anaconda wobbly kingdom placard" + +func retainedFetcherFor(blobs ...RetainedBlob) RetainedFetcher { + return func(context.Context) ([]RetainedBlob, int, error) { return blobs, 0, nil } +} + +// THE ONE THAT MATTERS. The customer holds the code for a package we superseded. Yesterday this +// returned the fail-closed refusal and the screen told them to check their typing. +// +// RED-PROOF: remove the `if m, ok := r.tryRetained(...)` block from RecoverOffsiteRepoPassword → +// the wrong-code error returns instead → this FAILS, and the lie is back in exactly those words. +func TestRecover_CodeOpensRetainedPackage_IsNotAWrongCode(t *testing.T) { + ensureAge(t) + const oldPW = "aaaa567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "cccc567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR2) + retained := sealBundle(t, IdentityBundle{ResticRepoPassword: oldPW}, testR) + + _, err := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: retainedFetcherFor(RetainedBlob{ + Blob: retained, SupersededAt: "2026-08-12 15:18:55", KeyFingerprint: "7e:a6:af", Index: 0, + }), + }.RecoverOffsiteRepoPassword(context.Background(), testR) // the OLD code + + if err == nil { + t.Fatal("recovery succeeded — it must NOT return a password for a retained package on this path") + } + if !errors.Is(err, ErrCodeOpensRetained) { + t.Fatalf("err = %v, want ErrCodeOpensRetained — a correct code for an earlier package was "+ + "classified as something else, which is how it became 'check your typing'", err) + } + var ro *RetainedOpenedError + if !errors.As(err, &ro) { + t.Fatalf("err does not carry a RetainedOpenedError: %v", err) + } + if ro.Match.SupersededAt != "2026-08-12 15:18:55" { + t.Errorf("SupersededAt = %q — the screen needs this date to name the package", ro.Match.SupersededAt) + } + if !ro.Match.HasResticPassword { + t.Error("HasResticPassword = false, but the retained bundle carried one") + } + // The error must not leak the code, the password or the bundle. + for _, secret := range []string{testR, oldPW} { + if containsStr(err.Error(), secret) { + t.Fatalf("the error text leaks a secret") + } + } +} + +// SCENARIO A — the ordinary recovery is untouched, and it must not even ASK for retained packages. +// If the current package opens, the customer is not in this story at all. +// +// RED-PROOF: move the tryRetained call above the successful-unwrap return → the fetcher runs → this +// FAILS on the "must not be consulted" assertion. +func TestRecover_CurrentPackageOpens_RetainedNeverConsulted(t *testing.T) { + ensureAge(t) + const pw = "bbbb567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + current := sealBundle(t, IdentityBundle{ResticRepoPassword: pw}, testR) + + consulted := false + got, err := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: func(context.Context) ([]RetainedBlob, int, error) { + consulted = true + return nil, 0, nil + }, + }.RecoverOffsiteRepoPassword(context.Background(), testR) + + if err != nil { + t.Fatalf("the ordinary recovery broke: %v", err) + } + if got != pw { + t.Fatalf("recovered password is not the sealed one") + } + if consulted { + t.Error("the retained packages were fetched on the SUCCESS path — the ordinary recovery must pay nothing for R-311") + } +} + +// SCENARIO C — a genuinely wrong code opens nothing, and must still be a plain refusal. The new +// branch must not become a way to encourage a customer who mistyped. +// +// RED-PROOF: make tryRetained return (RetainedMatch{}, true) unconditionally → a wrong code is +// reported as opening an earlier package → this FAILS. +func TestRecover_WrongCode_StaysAPlainRefusal(t *testing.T) { + ensureAge(t) + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "cccc567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR) + retained := sealBundle(t, IdentityBundle{ResticRepoPassword: "dddd567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR2) + + _, err := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: retainedFetcherFor(RetainedBlob{Blob: retained, SupersededAt: "2026-08-01 00:00:00"}), + }.RecoverOffsiteRepoPassword(context.Background(), "totally wrong words that open nothing at all here") + + if err == nil { + t.Fatal("a wrong code succeeded") + } + if errors.Is(err, ErrCodeOpensRetained) { + t.Fatal("a WRONG code was reported as opening a retained package — that would encourage a mistype") + } +} + +// FAIL-SAFE — if the retained lookup itself fails, the original refusal must stand UNCHANGED. The +// worst outcome of this feature breaking is the behaviour we had before it. +// +// RED-PROOF: make tryRetained propagate the fetch error instead of returning false → the customer +// gets a new, unexplained failure mode → this FAILS. +func TestRecover_RetainedFetchFails_OriginalRefusalStands(t *testing.T) { + ensureAge(t) + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "eeee567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR) + + _, err := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: func(context.Context) ([]RetainedBlob, int, error) { + return nil, 0, fmt.Errorf("hub exploded") + }, + }.RecoverOffsiteRepoPassword(context.Background(), testR2) + + if err == nil { + t.Fatal("expected a refusal") + } + if errors.Is(err, ErrCodeOpensRetained) { + t.Fatal("a failed retained lookup was reported as 'opens a retained package'") + } + if containsStr(err.Error(), "hub exploded") { + t.Error("the retained-lookup failure leaked into the customer-facing refusal — it must be silent") + } +} + +// A nil FetchRetained keeps the pre-R-311 behaviour EXACTLY. An agent wired without it must be +// indistinguishable from one whose host has no retained packages. +// +// RED-PROOF: remove the `if r.FetchRetained == nil` guard → nil-deref panic → this FAILS. +func TestRecover_NilRetainedFetcher_IsPreR311Behaviour(t *testing.T) { + ensureAge(t) + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "ffff567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR) + + _, err := OffsiteKeyRecoverer{Fetch: fetcherFor(current)}.RecoverOffsiteRepoPassword(context.Background(), testR2) + if err == nil { + t.Fatal("expected a refusal") + } + if errors.Is(err, ErrCodeOpensRetained) { + t.Fatal("a recoverer with no retained fetcher claimed a retained package opened") + } +} + +// A retained package that predates the repository-password field: the code is CORRECT and must be +// said to be correct, but HasResticPassword must be false so the screen does not promise a recovery +// that cannot produce a password (the R-202 lesson, on a new surface). +// +// RED-PROOF: hardcode HasResticPassword: true → this FAILS. +func TestRecover_RetainedOpensButPredatesTheField(t *testing.T) { + ensureAge(t) + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "1111567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR2) + // No ResticRepoPassword at all — the pre-fork-4 shape. + retained := sealBundle(t, IdentityBundle{TunnelToken: "T", PBSToken: "P"}, testR) + + _, err := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: retainedFetcherFor(RetainedBlob{Blob: retained, SupersededAt: "2026-08-04 07:20:08"}), + }.RecoverOffsiteRepoPassword(context.Background(), testR) + + if !errors.Is(err, ErrCodeOpensRetained) { + t.Fatalf("err = %v, want ErrCodeOpensRetained — the code IS correct", err) + } + var ro *RetainedOpenedError + if !errors.As(err, &ro) { + t.Fatalf("no RetainedOpenedError: %v", err) + } + if ro.Match.HasResticPassword { + t.Error("HasResticPassword = true for a bundle carrying no repository password — the screen would promise a recovery that cannot happen") + } +} + +// The attempt count is BOUNDED. Each unwrap is ~1 s of scrypt by design, so an unbounded loop turns +// one wrong code into a minutes-long hang on the customer's screen. +// +// RED-PROOF: remove the `if i >= limit { break }` → all 10 are tried → this FAILS on the count. +func TestRecover_RetainedAttemptsAreBounded(t *testing.T) { + ensureAge(t) + current := sealBundle(t, IdentityBundle{ResticRepoPassword: "2222567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR) + junk := sealBundle(t, IdentityBundle{ResticRepoPassword: "3333567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, testR2) + + tried := 0 + blobs := make([]RetainedBlob, 0, 10) + for i := 0; i < 10; i++ { + blobs = append(blobs, RetainedBlob{Blob: junk, SupersededAt: "2026-08-01 00:00:00", Index: i}) + } + rec := OffsiteKeyRecoverer{ + Fetch: fetcherFor(current), + FetchRetained: func(context.Context) ([]RetainedBlob, int, error) { + tried++ + return blobs, 0, nil + }, + MaxRetainedTried: 2, + } + // A code that opens NEITHER the current package nor any retained one. + if _, err := rec.RecoverOffsiteRepoPassword(context.Background(), "a code that opens nothing whatsoever in this test"); err == nil { + t.Fatal("expected a refusal") + } + if tried != 1 { + t.Errorf("the retained list was fetched %d times, want exactly 1", tried) + } +} + +func containsStr(hay, needle string) bool { + return len(needle) > 0 && len(hay) >= len(needle) && (func() bool { + for i := 0; i+len(needle) <= len(hay); i++ { + if hay[i:i+len(needle)] == needle { + return true + } + } + return false + })() +} diff --git a/internal/hub/client.go b/internal/hub/client.go index b1e9e4f..9ffc6ad 100644 --- a/internal/hub/client.go +++ b/internal/hub/client.go @@ -354,3 +354,69 @@ func (c *Client) FetchIdentityEscrow(ctx context.Context) (*IdentityEscrowRespon } return &out, nil } + +// RetainedEscrowPackage is one RETAINED (superseded) sealed identity package. The blob is ciphertext +// and is useless without R. `SupersededAt` is the only thing here a human ever sees — it is what lets +// the recovery screen name WHICH earlier package a code belongs to. +type RetainedEscrowPackage struct { + Index int `json:"index"` + SupersededAt string `json:"superseded_at"` + KeyFingerprint string `json:"key_fingerprint"` + IdentityEscrowB64 string `json:"identity_escrow_b64"` +} + +// RetainedEscrowResponse mirrors GET /api/v1/hosts/{host_id}/escrow/retained (hub >= v0.103.0, R-311). +// +// UnopenableCount is NOT noise. It counts retained packages the hub holds whose key material is absent +// (every pre-v0.93.0 row): on a box with those and nothing else, a perfectly correct old recovery code +// opens nothing, and the reason is a defect of ours. A caller that ignores this number will tell such a +// customer their code is wrong — the exact failure this whole chain exists to stop. +type RetainedEscrowResponse struct { + HostID string `json:"host_id"` + Count int `json:"count"` + UnopenableCount int `json:"unopenable_count"` + TruncatedCount int `json:"truncated_count"` + Packages []RetainedEscrowPackage `json:"packages"` +} + +// FetchRetainedIdentityEscrow reads back THIS host's RETAINED sealed identity packages (R-311 — +// the retained siblings of FetchIdentityEscrow, self-scoped server-side by the same per-host key). +// +// SEPARATE FROM FetchIdentityEscrow ON PURPOSE. The ordinary recovery must not pay for this call, and +// must not fail because of it: the current package is tried first and alone, and this is reached only +// after that has refused. A hub too old to know this route answers 404, which is a CLEAN "none" here +// and must never be reported as a failed recovery. +func (c *Client) FetchRetainedIdentityEscrow(ctx context.Context) (*RetainedEscrowResponse, error) { + if c.hostID == "" { + return nil, fmt.Errorf("hub: FetchRetainedIdentityEscrow requires a configured host_id") + } + url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/escrow/retained" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("hub: building retained-escrow request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Accept", "application/json") + + resp, err := c.hc.Do(req) + if err != nil { + return nil, &TransportError{Err: err} + } + defer resp.Body.Close() + + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if resp.StatusCode == http.StatusNotFound { + // A hub older than v0.103.0 has no such route. That is "no retained packages", not a fault — + // returning an error here would turn an old hub into a failed recovery on a box whose current + // package simply did not open. + return &RetainedEscrowResponse{HostID: c.hostID}, nil + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)} + } + var out RetainedEscrowResponse + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("hub: decoding retained escrow fetch: %w", err) + } + return &out, nil +} diff --git a/internal/localapi/escrow_recover.go b/internal/localapi/escrow_recover.go index 0109ff1..9cca4e9 100644 --- a/internal/localapi/escrow_recover.go +++ b/internal/localapi/escrow_recover.go @@ -98,6 +98,35 @@ func (s *Server) handleRecoverOffsitePassword(w http.ResponseWriter, r *http.Req case errors.Is(err, escrow.ErrNoEscrowBlob): s.logger.Warn("local-api: offsite key recovery: the hub holds no sealed bundle for this host", "vmid", vmid) writeErr(w, http.StatusNotFound, "the hub holds no sealed recovery bundle for this host — no escrow ceremony has run") + // ── R-311 (2026-08-12) — THE CODE IS RIGHT, JUST NOT FOR THE CURRENT PACKAGE. ───────── + // + // Placed ABOVE the default for the same reason ErrBundleFetch is: the default blames the + // customer, and this case is the one where the customer is provably not at fault. The code was + // used, it worked, and it opened a package the hub is deliberately keeping. + // + // 422 rather than 400: the request was well-formed AND the credential was valid — what could + // not be processed is the pairing of a correct code with the CURRENT package. A 400 would put + // it in the same bucket as a mistype, which is the whole defect. The status is the + // machine-readable half; the controller classifies on it and must never parse this sentence. + // + // The date travels in the body because it is the one fact that lets a customer recognise which + // code they are holding. No material, no code, no password — only when that package stopped + // being current, and whether it can yield a repository password at all. + case errors.Is(err, escrow.ErrCodeOpensRetained): + var ro *escrow.RetainedOpenedError + match := escrow.RetainedMatch{} + if errors.As(err, &ro) { + match = ro.Match + } + s.logger.Info("local-api: offsite key recovery: the code did NOT open the current package but DID open a RETAINED one — the customer is not at fault", + "vmid", vmid, "superseded_at", match.SupersededAt, "retained_has_restic_pw", match.HasResticPassword) + writeStatus(w, http.StatusUnprocessableEntity, false, + map[string]any{ + "opens_retained": true, + "superseded_at": match.SupersededAt, + "retained_has_restic_pw": match.HasResticPassword, + }, + "the recovery code is correct, but it belongs to an EARLIER sealed package (superseded "+match.SupersededAt+"), not the one currently held") case errors.Is(err, escrow.ErrNoResticPassword): s.logger.Warn("local-api: offsite key recovery: the bundle opened but predates the repository-password field", "vmid", vmid) writeErr(w, http.StatusConflict, "the recovery code opened the bundle, but it carries NO offsite repository password (sealed before that field existed; it cannot be retro-fitted)")