4ed938cce4
The recovery unit on the customer's drive now carries the PORTABLE secret class, so Tier-1/Tier-2 restore no longer depends on the whole-guest tier. A customer needs the drive and nothing else. Part 0's rulings overturned the brief's recommendation, on evidence: - the data_key flag is untrustworthy (4+ encryption keys the catalog itself labels as such are unflagged) -> R-127 - a DB password is not resettable in practice: POSTGRES_PASSWORD is ignored once PGDATA is non-empty, so a regenerated value leaves the app unable to authenticate against its own restored rows while the dump replay still reports success (proven on a throwaway postgres:16-alpine) Ruling (operator): type:secret travels, type:password never does, minus the nonPortableSecrets code register. Plaintext -- withholding the internet- reachable class is what licenses that, and the two are coupled. Precedence: the UNIT WINS over the guest -- the unit's secrets were captured in the same run as the dumps beside them, so they match the data being restored. The fail-closed data-key gate is unchanged. Secret values are never logged; the manifest records NAMES only.
377 lines
16 KiB
Go
377 lines
16 KiB
Go
package backup
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// captureFixtureUnit writes a real stack tree, runs the REAL CaptureRecoveryUnit over it, and returns
|
|
// the drive (namespace root) holding the resulting unit.
|
|
//
|
|
// Fixtures come from a CAPTURED unit rather than hand-written YAML deliberately: the capture side and
|
|
// the restore side must meet at real bytes on a real filesystem, so a change to the on-disk shape
|
|
// (header text, key ordering, the non-secret/portable split) cannot pass by having a test agree with
|
|
// itself. Everything from buildUnitAppYaml through readUnitEnv is production code here.
|
|
func captureFixtureUnit(t *testing.T, portable map[string]string) (drive string) {
|
|
t.Helper()
|
|
tmp := t.TempDir()
|
|
stackDir := filepath.Join(tmp, "stack")
|
|
drive = filepath.Join(tmp, "drive")
|
|
if err := os.MkdirAll(stackDir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mustWrite(t, filepath.Join(stackDir, "docker-compose.yml"),
|
|
"services:\n app:\n image: example/app:1\n db:\n image: postgres:16\n")
|
|
mustWrite(t, filepath.Join(stackDir, ".felhom.yml"), "display_name: App\n")
|
|
mustWrite(t, filepath.Join(stackDir, "app.yaml"), "deployed: true\nenv:\n SUBDOMAIN: trips\n")
|
|
|
|
var names []string
|
|
for _, n := range []string{"DB_PASSWORD", "SECRET_KEY"} {
|
|
if _, ok := portable[n]; ok {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
info := RecoveryInfo{
|
|
StackDir: stackDir,
|
|
DisplayName: "App",
|
|
ImagePins: []string{"example/app:1"},
|
|
NonSecretEnv: map[string]string{"SUBDOMAIN": "trips"},
|
|
SecretEnvVars: []string{"DB_PASSWORD", "SECRET_KEY"},
|
|
DataKeyEnvVars: []string{"SECRET_KEY"},
|
|
PortableSecretEnvVars: names,
|
|
PortableSecrets: portable,
|
|
}
|
|
m := &Manager{
|
|
logger: log.New(io.Discard, "", 0),
|
|
systemDataPath: filepath.Join(tmp, "system"),
|
|
stackProvider: &fakeRecoveryProvider{info: info, hdd: drive},
|
|
version: "vtest",
|
|
}
|
|
if err := m.CaptureRecoveryUnit("app"); err != nil {
|
|
t.Fatalf("capture fixture: %v", err)
|
|
}
|
|
return drive
|
|
}
|
|
|
|
// TestRestoreFromRecoveryUnitWithGuestAbsent is D5's entire claim, as a test rather than a
|
|
// description: a Tier-1/2 restore SUCCEEDS when the guest's app.yaml is unavailable.
|
|
//
|
|
// SEAM (R-125): injection is at Manager.stackProvider only — i.e. the docker/compose operations and the
|
|
// guest's app.yaml decrypt. RecoverStackSecrets returning nil IS the guest being gone: it is exactly
|
|
// what the real adapter returns when the stack or its app.yaml cannot be read (main.go
|
|
// GetStack/LoadAppConfigDecrypted nil paths). Everything under test is production code: the unit was
|
|
// written by the real CaptureRecoveryUnit, read back by the real readUnitEnv, and reconciled by the
|
|
// real reconcileRestoreSecrets.
|
|
func TestRestoreFromRecoveryUnitWithGuestAbsent(t *testing.T) {
|
|
const (
|
|
dataKey = "deadbeefdeadbeef"
|
|
dbPw = "pw-from-the-drive"
|
|
)
|
|
drive := captureFixtureUnit(t, map[string]string{"DB_PASSWORD": dbPw, "SECRET_KEY": dataKey})
|
|
|
|
// The guest is GONE: no secrets recoverable from it at all.
|
|
fake := &fakeRecoveryProvider{hdd: drive, running: true, secrets: nil}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0),
|
|
systemDataPath: filepath.Join(drive, "..", "sys"), stackProvider: fake}
|
|
|
|
if err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
|
t.Fatalf("restore must succeed from the drive alone, got: %v", err)
|
|
}
|
|
if fake.gotEnv == nil {
|
|
t.Fatal("recreate was not called — the restore did not reach the redeploy")
|
|
}
|
|
// The consequence: the app is redeployed with the key that decrypts the data beside it.
|
|
if fake.gotEnv["SECRET_KEY"] != dataKey {
|
|
t.Errorf("data-encrypting key not recovered from the unit: %q", fake.gotEnv["SECRET_KEY"])
|
|
}
|
|
if fake.gotEnv["DB_PASSWORD"] != dbPw {
|
|
t.Errorf("DB password not recovered from the unit: %q", fake.gotEnv["DB_PASSWORD"])
|
|
}
|
|
if fake.gotEnv["SUBDOMAIN"] != "trips" {
|
|
t.Errorf("plain config lost: %v", fake.gotEnv)
|
|
}
|
|
}
|
|
|
|
// TestRestoreFromRecoveryUnitGuestAbsentStillFailsClosed proves D5 did not soften the gate: with the
|
|
// data key in NEITHER the unit nor the guest, the restore still REFUSES and mutates nothing.
|
|
func TestRestoreFromRecoveryUnitGuestAbsentStillFailsClosed(t *testing.T) {
|
|
// Unit carries only the DB password — the data key is absent from both sources.
|
|
drive := captureFixtureUnit(t, map[string]string{"DB_PASSWORD": "pw"})
|
|
fake := &fakeRecoveryProvider{hdd: drive, running: true, secrets: nil}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0),
|
|
systemDataPath: filepath.Join(drive, "..", "sys"), stackProvider: fake}
|
|
|
|
err := m.RestoreFromRecoveryUnit("app")
|
|
if err == nil {
|
|
t.Fatal("expected fail-closed refusal when the data key is in neither source")
|
|
}
|
|
if !strings.Contains(err.Error(), "SECRET_KEY") {
|
|
t.Errorf("refusal should name the missing data key, got: %v", err)
|
|
}
|
|
if fake.gotEnv != nil {
|
|
t.Errorf("recreate must NOT be called on refusal, got %v", fake.gotEnv)
|
|
}
|
|
if fake.stopped {
|
|
t.Error("the live app must not be stopped when the restore refuses")
|
|
}
|
|
}
|
|
|
|
// TestRestoreFromRecoveryUnitOrchestration exercises the full in-process flow: read manifest →
|
|
// recover secrets → apply gate → recreate with the reconciled env. It proves (a) on success the
|
|
// recreate is called with non-secret env + recovered secrets merged, and (b) on a missing data-key the
|
|
// restore is REFUSED and recreate is never called.
|
|
func TestRestoreFromRecoveryUnitOrchestration(t *testing.T) {
|
|
newUnit := func(t *testing.T) (drive string) {
|
|
tmp := t.TempDir()
|
|
drive = filepath.Join(tmp, "drive")
|
|
// A schema-1 unit: no portable secrets, so the guest is the only source (pre-D5 behaviour).
|
|
mustWrite(t, filepath.Join(RecoveryUnitComposePath(drive, "app"), "app.yaml"),
|
|
"deployed: true\nenv:\n SUBDOMAIN: trips\n")
|
|
man := &RecoveryManifest{SchemaVersion: 1, AppName: "app", ControllerVer: "v",
|
|
SecretEnvVars: []string{"DB_PASSWORD", "SECRET_KEY"}, DataKeyEnvVars: []string{"SECRET_KEY"}}
|
|
if err := writeManifest(RecoveryUnitManifestPath(drive, "app"), man); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return drive
|
|
}
|
|
|
|
t.Run("success — recreate called with merged env", func(t *testing.T) {
|
|
drive := newUnit(t)
|
|
fake := &fakeRecoveryProvider{
|
|
hdd: drive,
|
|
running: true, // so the post-restore health wait returns promptly
|
|
secrets: map[string]string{"DB_PASSWORD": "pw", "SECRET_KEY": "deadbeef"},
|
|
}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0), systemDataPath: filepath.Join(drive, "..", "sys"),
|
|
stackProvider: fake}
|
|
if err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
|
t.Fatalf("restore: %v", err)
|
|
}
|
|
if fake.gotEnv == nil {
|
|
t.Fatal("recreate was not called")
|
|
}
|
|
if fake.gotEnv["SUBDOMAIN"] != "trips" || fake.gotEnv["DB_PASSWORD"] != "pw" || fake.gotEnv["SECRET_KEY"] != "deadbeef" {
|
|
t.Errorf("recreate got wrong env: %v", fake.gotEnv)
|
|
}
|
|
if !fake.stopped {
|
|
t.Errorf("app should be stopped before restore")
|
|
}
|
|
})
|
|
|
|
t.Run("schema-1 unit still restores from the guest — no regression", func(t *testing.T) {
|
|
// An old unit carries nothing; the guest must still be able to supply everything.
|
|
drive := newUnit(t)
|
|
fake := &fakeRecoveryProvider{hdd: drive, running: true,
|
|
secrets: map[string]string{"DB_PASSWORD": "pw", "SECRET_KEY": "deadbeef"}}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0),
|
|
systemDataPath: filepath.Join(drive, "..", "sys"), stackProvider: fake}
|
|
if err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
|
t.Fatalf("a pre-D5 unit must still restore: %v", err)
|
|
}
|
|
if fake.gotEnv["SECRET_KEY"] != "deadbeef" {
|
|
t.Errorf("guest fallback lost the data key: %v", fake.gotEnv)
|
|
}
|
|
})
|
|
|
|
t.Run("data-key unrecoverable — REFUSED, recreate not called", func(t *testing.T) {
|
|
drive := newUnit(t)
|
|
fake := &fakeRecoveryProvider{
|
|
hdd: drive,
|
|
secrets: map[string]string{"DB_PASSWORD": "pw"}, // SECRET_KEY (data_key) missing
|
|
}
|
|
m := &Manager{logger: log.New(io.Discard, "", 0), systemDataPath: filepath.Join(drive, "..", "sys"),
|
|
stackProvider: fake}
|
|
err := m.RestoreFromRecoveryUnit("app")
|
|
if err == nil {
|
|
t.Fatal("expected fail-closed refusal, got nil")
|
|
}
|
|
if fake.gotEnv != nil {
|
|
t.Errorf("recreate must NOT be called on refusal, got %v", fake.gotEnv)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestReconcileRestoreSecretsPrecedence pins the D5 precedence rule in BOTH directions. An undefined
|
|
// precedence between two sources of a decryption key is a data-loss bug waiting for its first
|
|
// disagreement, so this is not a style question.
|
|
//
|
|
// The UNIT wins: its secrets were captured in the same run as the dumps beside them, so the unit's
|
|
// value is the one that matches the data about to be restored. The guest's value is merely the most
|
|
// recent — and a rotated key does not decrypt data encrypted with the old one.
|
|
func TestReconcileRestoreSecretsPrecedence(t *testing.T) {
|
|
nonSecret := map[string]string{"SUBDOMAIN": "trips"}
|
|
names := []string{"DB_PASSWORD", "SECRET_KEY"}
|
|
dataKeys := []string{"SECRET_KEY"}
|
|
|
|
t.Run("both sources disagree — the UNIT wins", func(t *testing.T) {
|
|
unit := map[string]string{"DB_PASSWORD": "unit-pw", "SECRET_KEY": "unit-key"}
|
|
guest := map[string]string{"DB_PASSWORD": "guest-pw", "SECRET_KEY": "guest-key"}
|
|
full, missing, err := reconcileRestoreSecrets(nonSecret, unit, guest, names, dataKeys)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(missing) != 0 {
|
|
t.Errorf("missing: %v", missing)
|
|
}
|
|
if full["SECRET_KEY"] != "unit-key" {
|
|
t.Errorf("data key: got %q, want the UNIT's value (it matches the restored data)", full["SECRET_KEY"])
|
|
}
|
|
if full["DB_PASSWORD"] != "unit-pw" {
|
|
t.Errorf("DB password: got %q, want the UNIT's value (it matches the restored data dir hash)", full["DB_PASSWORD"])
|
|
}
|
|
})
|
|
|
|
t.Run("unit silent — the GUEST fills in", func(t *testing.T) {
|
|
// The withheld class (admin logins) is never in the unit, so this direction must work too.
|
|
guest := map[string]string{"DB_PASSWORD": "guest-pw", "SECRET_KEY": "guest-key"}
|
|
full, _, err := reconcileRestoreSecrets(nonSecret, nil, guest, names, dataKeys)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if full["SECRET_KEY"] != "guest-key" || full["DB_PASSWORD"] != "guest-pw" {
|
|
t.Errorf("guest fallback not applied: %v", full)
|
|
}
|
|
})
|
|
|
|
t.Run("unit present but EMPTY for a name — the guest fills in", func(t *testing.T) {
|
|
// An empty value is not a value; it must not shadow a good one from the guest.
|
|
unit := map[string]string{"SECRET_KEY": ""}
|
|
guest := map[string]string{"SECRET_KEY": "guest-key", "DB_PASSWORD": "guest-pw"}
|
|
full, _, err := reconcileRestoreSecrets(nonSecret, unit, guest, names, dataKeys)
|
|
if err != nil {
|
|
t.Fatalf("an empty unit value must fall through to the guest, got: %v", err)
|
|
}
|
|
if full["SECRET_KEY"] != "guest-key" {
|
|
t.Errorf("empty unit value shadowed the guest: %q", full["SECRET_KEY"])
|
|
}
|
|
})
|
|
|
|
t.Run("a portable secret never shadows plain config", func(t *testing.T) {
|
|
// GetStackRecoveryInfo keeps the two sets disjoint; if that ever breaks, the merge order in
|
|
// buildUnitAppYaml decides silently. Pin the intended outcome.
|
|
full, _, err := reconcileRestoreSecrets(map[string]string{"DB_PASSWORD": "should-not-win"},
|
|
map[string]string{"DB_PASSWORD": "unit-pw"}, nil, []string{"DB_PASSWORD"}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if full["DB_PASSWORD"] != "unit-pw" {
|
|
t.Errorf("the secret source must win over a stray non-secret entry: %q", full["DB_PASSWORD"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestReconcileRestoreSecrets covers the safety-critical fail-closed gate + secret reconciliation.
|
|
func TestReconcileRestoreSecrets(t *testing.T) {
|
|
nonSecret := map[string]string{"SUBDOMAIN": "trips", "DOMAIN": "demo-felhom.eu"}
|
|
|
|
t.Run("all recovered, no data_key — full env, no error", func(t *testing.T) {
|
|
guest := map[string]string{"DB_PASSWORD": "pw", "SECRET_KEY": "deadbeef"}
|
|
full, missing, err := reconcileRestoreSecrets(nonSecret, nil, guest,
|
|
[]string{"DB_PASSWORD", "SECRET_KEY"}, nil)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(missing) != 0 {
|
|
t.Errorf("missing: %v", missing)
|
|
}
|
|
// Non-secret + both secrets present, and recovered values used VERBATIM (regenerate nothing).
|
|
if full["SUBDOMAIN"] != "trips" || full["DB_PASSWORD"] != "pw" || full["SECRET_KEY"] != "deadbeef" {
|
|
t.Errorf("full env wrong: %v", full)
|
|
}
|
|
})
|
|
|
|
t.Run("data_key missing from BOTH sources — FAIL CLOSED (refuse)", func(t *testing.T) {
|
|
guest := map[string]string{"DB_PASSWORD": "pw"} // SECRET_KEY (a data_key) is gone
|
|
full, _, err := reconcileRestoreSecrets(nonSecret, nil, guest,
|
|
[]string{"DB_PASSWORD", "SECRET_KEY"}, []string{"SECRET_KEY"})
|
|
if err == nil {
|
|
t.Fatal("expected fail-closed error for missing data-encrypting key, got nil")
|
|
}
|
|
if full != nil {
|
|
t.Errorf("full env should be nil on refusal, got %v", full)
|
|
}
|
|
})
|
|
|
|
t.Run("data_key empty in both — FAIL CLOSED", func(t *testing.T) {
|
|
guest := map[string]string{"SECRET_KEY": ""} // present but empty == unrecoverable
|
|
_, _, err := reconcileRestoreSecrets(nonSecret, map[string]string{"SECRET_KEY": ""}, guest,
|
|
[]string{"SECRET_KEY"}, []string{"SECRET_KEY"})
|
|
if err == nil {
|
|
t.Fatal("empty data-key value must fail closed")
|
|
}
|
|
})
|
|
|
|
t.Run("data_key recovered from the UNIT — no refusal", func(t *testing.T) {
|
|
// The D5 case: the guest is gone but the unit carries the key, so the gate must NOT fire.
|
|
unit := map[string]string{"SECRET_KEY": "deadbeef", "DB_PASSWORD": "pw"}
|
|
full, missing, err := reconcileRestoreSecrets(nonSecret, unit, nil,
|
|
[]string{"DB_PASSWORD", "SECRET_KEY"}, []string{"SECRET_KEY"})
|
|
if err != nil {
|
|
t.Fatalf("the unit's data key must satisfy the gate: %v", err)
|
|
}
|
|
if len(missing) != 0 {
|
|
t.Errorf("nothing should be missing: %v", missing)
|
|
}
|
|
if full["SECRET_KEY"] != "deadbeef" {
|
|
t.Errorf("data key wrong: %v", full)
|
|
}
|
|
})
|
|
|
|
t.Run("resettable secret missing — proceed with warning", func(t *testing.T) {
|
|
guest := map[string]string{"SECRET_KEY": "deadbeef"} // data_key ok; DB_PASSWORD missing
|
|
full, missing, err := reconcileRestoreSecrets(nonSecret, nil, guest,
|
|
[]string{"DB_PASSWORD", "SECRET_KEY"}, []string{"SECRET_KEY"})
|
|
if err != nil {
|
|
t.Fatalf("a missing resettable secret must NOT fail closed: %v", err)
|
|
}
|
|
if len(missing) != 1 || missing[0] != "DB_PASSWORD" {
|
|
t.Errorf("missing should be [DB_PASSWORD], got %v", missing)
|
|
}
|
|
if full["SECRET_KEY"] != "deadbeef" {
|
|
t.Errorf("data-key should be preserved verbatim: %v", full)
|
|
}
|
|
if _, present := full["DB_PASSWORD"]; present {
|
|
t.Errorf("missing resettable secret should be absent, not regenerated")
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestReadUnitEnvSplitsByManifest proves the split is driven by the manifest's portable names, and that
|
|
// a schema-1 unit (no names) degrades to "everything is plain config" rather than losing entries.
|
|
func TestReadUnitEnvSplitsByManifest(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "app.yaml")
|
|
mustWrite(t, path, "deployed: true\nenv:\n SUBDOMAIN: trips\n DB_PASSWORD: pw\n SECRET_KEY: key\n")
|
|
|
|
t.Run("named secrets land in unitSecrets, the rest in nonSecret", func(t *testing.T) {
|
|
nonSecret, unitSecrets := readUnitEnv(path, []string{"DB_PASSWORD", "SECRET_KEY"})
|
|
if nonSecret["SUBDOMAIN"] != "trips" || len(nonSecret) != 1 {
|
|
t.Errorf("nonSecret = %v, want only SUBDOMAIN", nonSecret)
|
|
}
|
|
if unitSecrets["DB_PASSWORD"] != "pw" || unitSecrets["SECRET_KEY"] != "key" || len(unitSecrets) != 2 {
|
|
t.Errorf("unitSecrets = %v, want the two named secrets", unitSecrets)
|
|
}
|
|
})
|
|
|
|
t.Run("schema-1 (no portable names) — everything is plain config, nothing lost", func(t *testing.T) {
|
|
nonSecret, unitSecrets := readUnitEnv(path, nil)
|
|
if len(unitSecrets) != 0 {
|
|
t.Errorf("a schema-1 unit carries no secrets, got %v", unitSecrets)
|
|
}
|
|
if len(nonSecret) != 3 {
|
|
t.Errorf("nonSecret should keep every entry, got %v", nonSecret)
|
|
}
|
|
})
|
|
|
|
t.Run("absent file — empty maps, no panic", func(t *testing.T) {
|
|
nonSecret, unitSecrets := readUnitEnv(filepath.Join(dir, "nope.yaml"), []string{"DB_PASSWORD"})
|
|
if len(nonSecret) != 0 || len(unitSecrets) != 0 {
|
|
t.Errorf("want empty maps, got %v / %v", nonSecret, unitSecrets)
|
|
}
|
|
})
|
|
}
|