Files
felhom-agent/internal/escrow/recover_test.go
T
admin a2e914f683
gates / gates (push) Successful in 7s
v0.126.0: a fetch failure is not a wrong recovery code (R-224)
A hub the agent could not reach was reported to the customer as a bad recovery
code. Measured live 2026-08-05 (CAMPAIGN-11 F3): hub firewalled off, a CORRECT
current code, and the customer told it did not open their package — in 0.0556s
against ~1.0s for a real unseal. No unseal was attempted.

The discriminator existed here and this boundary threw it away: recover.go
fails at four distinguishable points and the local-api handler had cases for
two, with a default answering 'the recovery code did not open the sealed
bundle, OR the bundle could not be fetched'.

escrow.ErrBundleFetch now joins the fetch leg and the handler routes it to 502
with its own words — the code was NOT used. 502 not 4xx: the request was not
bad, an upstream dependency failed. Four situations, four statuses: 502 fetch /
400 fetched-and-refused / 404 no bundle / 409 predates the field. The
controller classifies on the STATUS and never parses the sentence.

A GREEN TEST NAMED THIS DEFECT AND DID NOT PREVENT IT.
TestRecoverOffsiteRepoPassword_FetchErrorIsDistinct has said since v0.125.0
that the operator must not be sent to re-read their code because the hub was
unreachable — and passed throughout, because it asserted this package's error
STRING one layer below the merge, and a string is not something a caller can
branch on. Re-pointed at the sentinel, with a consequence-level twin asserting
the status.

Red-proofs: removing the %w join fails the sentinel test; deleting the handler
case makes fetch and wrong-code both answer 400 with the wrong-code sentence.

29 packages ok, vet clean, agent gates OK.
2026-08-06 07:55:15 +02:00

268 lines
13 KiB
Go

package escrow
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
// R-199 links 6→8, with REAL crypto (age is present on the build/demo host; ensureAge skips
// elsewhere). These are the unit half of the session's question — "is the repository password
// actually recoverable from the sealed bundle" — and the live half is the same equality on hardware.
const testR = "correct horse battery staple sedative anaconda wobbly kingdom placard yodel"
func sealBundle(t *testing.T, b IdentityBundle, r string) []byte {
t.Helper()
blob, err := WrapIdentityBundle(context.Background(), b, r)
if err != nil {
t.Fatalf("WrapIdentityBundle: %v", err)
}
return blob
}
func fetcherFor(blob []byte) BlobFetcher {
return func(context.Context) ([]byte, bool, error) { return blob, true, nil }
}
// Scenario A (unit) — the recovered repository password is BYTE-IDENTICAL to the sealed one, and it
// is the REPOSITORY password rather than some other field of a bundle that also parses.
//
// RED-PROOF: return bundle.PBSToken (or TunnelToken, or WGPrivateKey) instead of
// bundle.ResticRepoPassword → a plausible-looking bundle yields a non-matching key → this FAILS.
// That mutation is the shape of the bug that would otherwise ship silently, because every one of
// those fields is a non-empty string that looks like a secret.
func TestRecoverOffsiteRepoPassword_ReturnsTheRepositoryPassword(t *testing.T) {
ensureAge(t)
const repoPW = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
blob := sealBundle(t, IdentityBundle{
TunnelToken: "TUNNEL-TOKEN-NOT-THE-ANSWER",
PBSToken: "PBS-TOKEN-NOT-THE-ANSWER",
WGPrivateKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
ResticRepoPassword: repoPW,
}, testR)
got, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR)
if err != nil {
t.Fatalf("recover: %v", err)
}
if got != repoPW {
t.Fatalf("the recovered key is not the sealed repository password (len %d vs %d) — a different "+
"field of the bundle was returned", len(got), len(repoPW))
}
// Belt: it must not be any of the OTHER fields, so a future refactor cannot satisfy the check
// above by coincidence.
for _, other := range []string{"TUNNEL-TOKEN-NOT-THE-ANSWER", "PBS-TOKEN-NOT-THE-ANSWER", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="} {
if got == other {
t.Fatalf("the recoverer returned the wrong bundle field")
}
}
}
// Scenario B — a WRONG recovery code fails closed, the failure names no secret, and nothing is
// written. The fail-closed property is the crypto's (age's scrypt KDF), which is why there is no
// validation step here to get wrong — the test pins that it stays that way.
func TestRecoverOffsiteRepoPassword_WrongCodeFailsClosed(t *testing.T) {
ensureAge(t)
const repoPW = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p", ResticRepoPassword: repoPW}, testR)
got, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), "not the recovery code at all")
if err == nil {
t.Fatal("a wrong recovery code MUST fail — a plausible-but-wrong bundle is the one outcome the design forbids")
}
if got != "" {
t.Fatalf("a failed unseal returned %d bytes — there must be no partial result", len(got))
}
// The error may name the step; it may never name a secret.
for _, secret := range []string{repoPW, testR, "not the recovery code at all"} {
if strings.Contains(err.Error(), secret) {
t.Fatalf("the failure message leaked a secret: %v", err)
}
}
}
// A bundle with no repository password is its OWN answer, not a wrong-code error. Sealed before
// fork-4 (agent < v0.77.0) the field did not exist; sending the operator to re-check a correctly
// typed recovery code would be the wrong instruction.
func TestRecoverOffsiteRepoPassword_PreForkFourBundle(t *testing.T) {
ensureAge(t)
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p"}, testR)
_, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR)
if !errors.Is(err, ErrNoResticPassword) {
t.Fatalf("a pre-fork-4 bundle must report its own error, got %v", err)
}
}
// Scenario D at this layer — no blob is a clean, distinguishable answer.
func TestRecoverOffsiteRepoPassword_NoBlob(t *testing.T) {
rec := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, nil }}
_, err := rec.RecoverOffsiteRepoPassword(context.Background(), testR)
if !errors.Is(err, ErrNoEscrowBlob) {
t.Fatalf("absent blob must yield ErrNoEscrowBlob, got %v", err)
}
}
// Scenario F — R persists NOWHERE. TMPDIR is redirected into the test's own directory, the unseal is
// run for real, and the whole tree is then walked: no file may contain R (or the recovered password),
// and the staging directory the unseal creates must be gone.
//
// RED-PROOF: write R to a temp file anywhere in the flow (e.g. add
// `os.WriteFile(filepath.Join(work,"r"), []byte(recoveryCode), 0o600)` inside UnwrapIdentity before
// its defer removes the dir — or simply drop that defer and let the plaintext staging survive) → the
// walk finds it → this FAILS.
func TestRecoverOffsiteRepoPassword_RLeavesNoTrace(t *testing.T) {
ensureAge(t)
const repoPW = "1111111111111111111111111111111111111111111111111111111111111111"
tmp := t.TempDir()
t.Setenv("TMPDIR", tmp) // os.MkdirTemp honours this — every staging dir lands under the walk
const wrongR = "wrong code entirely"
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p", ResticRepoPassword: repoPW}, testR)
if _, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR); err != nil {
t.Fatalf("recover: %v", err)
}
// A failed unseal must leave nothing either — exercise both paths before walking.
_, _ = (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), wrongR)
// THE PRIMARY ASSERTION IS EMPTINESS, not content. A content scan alone is defeatable by a later
// call OVERWRITING the leaked file with a different secret — which is exactly how the first
// version of this test passed its own red-proof while R sat on disk. Nothing in this test writes
// under TMPDIR, so after both calls the tree must contain no files at all.
var survivors []string
err := filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
if err != nil || info == nil || info.IsDir() || path == tmp {
return nil
}
survivors = append(survivors, strings.TrimPrefix(path, tmp))
return nil
})
if err != nil {
t.Fatal(err)
}
if len(survivors) > 0 {
t.Fatalf("the unseal left %d file(s) behind under TMPDIR: %v — R, the sealed blob and the "+
"recovered plaintext all pass through there and none of them may outlive the call", len(survivors), survivors)
}
// Defence in depth: any secret that DOES appear anywhere is named, for every code used.
_ = filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
if err != nil || info == nil || info.IsDir() {
return nil
}
body, rerr := os.ReadFile(path)
if rerr != nil {
return nil
}
for label, secret := range map[string]string{"R": testR, "a wrong R": wrongR, "the repository password": repoPW} {
if strings.Contains(string(body), secret) {
t.Errorf("%s survived on disk at %s", label, path)
}
}
return nil
})
// And the staging directories are gone, not merely free of secrets.
entries, _ := os.ReadDir(tmp)
for _, e := range entries {
if e.IsDir() && strings.HasPrefix(e.Name(), "felhom-idesc-") {
t.Fatalf("an unseal staging directory survived: %s", e.Name())
}
}
}
// A fetch failure surfaces as a fetch failure, not as a wrong-code error — the operator must not be
// sent to re-read their recovery code because the hub was unreachable.
//
// ⚠ THIS TEST WAS GREEN THROUGHOUT THE DEFECT IT DESCRIBES (R-224, 2026-08-06). Its sentence is
// exactly right and it did not prevent anything, for two reasons worth keeping:
//
// 1. **It asserted the MECHANISM, one layer below the consequence.** It checked this package's error
// STRING. The merge happened one layer up, in the local-api handler's `default` branch, which
// answered a fetch failure with "the recovery code did not open the sealed bundle". The customer
// never sees this string; they see that one. The project's own rule — prefer the test that asserts
// the CONSEQUENCE (does the customer get blamed?) over the one that asserts the MECHANISM (is the
// error distinct here?) — names this case precisely.
// 2. **It asserted on TEXT.** `strings.Contains(err.Error(), …)` cannot be consumed by a caller, so
// it pinned something no production code could branch on. The distinction it checked was real and
// unusable.
//
// It now asserts the SENTINEL, which is what the handler branches on, and its consequence-level twin
// lives in `internal/localapi/escrow_recover_class_test.go` where the status is asserted.
func TestRecoverOffsiteRepoPassword_FetchErrorIsDistinct(t *testing.T) {
rec := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) {
return nil, false, errors.New("hub: connection refused")
}}
_, err := rec.RecoverOffsiteRepoPassword(context.Background(), testR)
if err == nil || !errors.Is(err, ErrBundleFetch) {
t.Fatalf("a fetch failure must classify as ErrBundleFetch, got %v", err)
}
if errors.Is(err, ErrNoEscrowBlob) || errors.Is(err, ErrNoResticPassword) {
t.Fatal("a transport failure must not masquerade as a content verdict")
}
}
// ── R-224 — A FAILED FETCH IS NOT A WRONG CODE ──────────────────────────────────────────────────
//
// CAMPAIGN-11 F3 measured the consequence of these two being indistinguishable: with the hub
// firewalled off and a CORRECT current recovery code, the customer was told the code did not open
// their package, in 0.0556 s — no unseal was attempted at all.
//
// The pair below is the whole point. Asserting only the first would pass with a `return ErrBundleFetch`
// stuck on every error path, which is the same defect pointing the other way.
func TestRecoverOffsiteRepoPassword_FetchFailureIsClassifiedAsFetch(t *testing.T) {
boom := errors.New("hub: transport error: dial tcp 37.191.56.193:443: connect: no route to host")
r := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, boom }}
_, err := r.RecoverOffsiteRepoPassword(context.Background(), testR)
if err == nil {
t.Fatal("a failing fetch must return an error")
}
// RED-PROOF: drop the `%w: %w` join in RecoverOffsiteRepoPassword (return the bare wrapped cause,
// as it was before R-224) → this FAILS, and the local-api handler falls back to the wrong-code
// message exactly as it did on 2026-08-05.
if !errors.Is(err, ErrBundleFetch) {
t.Fatalf("a failed fetch must classify as ErrBundleFetch, got %v", err)
}
// The underlying cause survives for the operator log.
if !errors.Is(err, boom) {
t.Fatalf("the fetch cause must stay wrapped for the operator, got %v", err)
}
// And it must NOT be mistaken for either of the bundle-content situations.
if errors.Is(err, ErrNoEscrowBlob) || errors.Is(err, ErrNoResticPassword) {
t.Fatalf("a transport failure is neither of the bundle-content errors: %v", err)
}
}
// The other half: a genuinely wrong code must NOT classify as a fetch failure, or the fix trades one
// misattribution for its mirror image and the customer is told the hub is down when they mistyped.
func TestRecoverOffsiteRepoPassword_WrongCodeIsNotAFetchFailure(t *testing.T) {
ensureAge(t)
blob := sealBundle(t, IdentityBundle{ResticRepoPassword: "0123456789abcdef"}, testR)
r := OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}
_, err := r.RecoverOffsiteRepoPassword(context.Background(),
"wrong horse battery staple sedative anaconda wobbly kingdom placard yodel")
if err == nil {
t.Fatal("a wrong recovery code must fail closed")
}
if errors.Is(err, ErrBundleFetch) {
t.Fatalf("a wrong code must NOT classify as a fetch failure, got %v", err)
}
}
// A clean "the hub holds nothing" keeps its own identity too — it is not a fetch failure, and the
// customer must not be told the hub was unreachable when it answered perfectly well.
func TestRecoverOffsiteRepoPassword_AbsentBlobIsNotAFetchFailure(t *testing.T) {
r := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, nil }}
_, err := r.RecoverOffsiteRepoPassword(context.Background(), testR)
if !errors.Is(err, ErrNoEscrowBlob) {
t.Fatalf("an absent blob must stay ErrNoEscrowBlob, got %v", err)
}
if errors.Is(err, ErrBundleFetch) {
t.Fatalf("an absent blob is not a fetch FAILURE, got %v", err)
}
}