376 lines
17 KiB
Go
376 lines
17 KiB
Go
package backup
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// D47 (operator ruling 2026-10-25 08:38; R-1269) — a new install never writes into an earlier install's unit.
|
|
//
|
|
// These tests replay the catalog sweep's order (`audits/catalog-sweep-2026-10-24/_unitprobe/tandoor/`): install A,
|
|
// a backup, „remove — keep backups", install B of the same version (new database password), then the five-minute
|
|
// refresh — and assert the CONSEQUENCE: the old unit's bytes, where they are, what a restore of it gets.
|
|
//
|
|
// COMPANION RED-PROOF (REPORT.md): on `e1613ae` (no guardUnitWrite) TestD47_TheSweepOrderNeverRewritesTheKeptUnit
|
|
// fails — the primary unit's app.yaml carries install B's password beside install A's data, and nothing is set aside.
|
|
|
|
const (
|
|
d47PwA = "PASSWORD-OF-INSTALL-A-the-data-was-made-with"
|
|
d47PwB = "PASSWORD-OF-INSTALL-B-a-fresh-reinstall"
|
|
)
|
|
|
|
// d47Info is the recovery info of one install of "tandoor" (a Postgres app on the system drive).
|
|
func d47Info(stackDir, drive, installID, pw string) RecoveryInfo {
|
|
return RecoveryInfo{
|
|
StackDir: stackDir,
|
|
DisplayName: "Tandoor",
|
|
ImagePins: []string{"example/tandoor:2.6.15"},
|
|
NonSecretEnv: map[string]string{"SUBDOMAIN": "recipes"},
|
|
SecretEnvVars: []string{"DB_PASSWORD"},
|
|
PortableSecretEnvVars: []string{"DB_PASSWORD"},
|
|
PortableSecrets: map[string]string{"DB_PASSWORD": pw},
|
|
InstallID: installID,
|
|
}
|
|
}
|
|
|
|
type d47Rig struct {
|
|
m *Manager
|
|
prov *fakeRecoveryProvider
|
|
drive string
|
|
stackDir string
|
|
logs *bytes.Buffer
|
|
sett *settings.Settings
|
|
}
|
|
|
|
// d47Setup: install A with a database dump and a volume tar, captured after its data run (the „Mentés most" order).
|
|
func d47Setup(t *testing.T, idA string) *d47Rig {
|
|
t.Helper()
|
|
tmp := t.TempDir()
|
|
r := &d47Rig{stackDir: filepath.Join(tmp, "stack"), drive: filepath.Join(tmp, "drive"), logs: &bytes.Buffer{}}
|
|
mustWrite(t, filepath.Join(r.stackDir, "docker-compose.yml"),
|
|
"services:\n tandoor:\n image: example/tandoor:2.6.15\n tandoor-postgres:\n image: postgres:17-alpine\n")
|
|
mustWrite(t, filepath.Join(r.stackDir, ".felhom.yml"), "display_name: Tandoor\n")
|
|
mustWrite(t, filepath.Join(r.stackDir, "app.yaml"), "deployed: true\nenv:\n SUBDOMAIN: recipes\n")
|
|
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.sett = sett
|
|
r.prov = &fakeRecoveryProvider{info: d47Info(r.stackDir, r.drive, idA, d47PwA), hdd: r.drive, running: true,
|
|
secrets: map[string]string{"DB_PASSWORD": d47PwA}}
|
|
r.m = &Manager{
|
|
logger: log.New(r.logs, "", 0),
|
|
systemDataPath: filepath.Join(tmp, "system"),
|
|
stackProvider: r.prov,
|
|
settings: sett,
|
|
version: "vtest",
|
|
}
|
|
mustWrite(t, filepath.Join(AppDBDumpPath(r.drive, "tandoor"), "tandoor-postgres.sql"), pgDump(1)+"-- the household's account\n")
|
|
mustWrite(t, filepath.Join(AppVolumeDumpPath(r.drive, "tandoor"), "tandoor_postgres_data.tar"), "DATA DIRECTORY OF INSTALL A")
|
|
if err := r.m.CaptureRecoveryUnit("tandoor"); err != nil {
|
|
t.Fatalf("install A's capture: %v", err)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// reinstall is „remove — keep backups" + a fresh install of the same version: a new identity and a new password.
|
|
func (r *d47Rig) reinstall(idB string) {
|
|
r.prov.info = d47Info(r.stackDir, r.drive, idB, d47PwB)
|
|
r.prov.secrets = map[string]string{"DB_PASSWORD": d47PwB}
|
|
}
|
|
|
|
func (r *d47Rig) unitDir() string { return RecoveryUnitPath(r.drive, "tandoor") }
|
|
|
|
func (r *d47Rig) setAside(t *testing.T) []string {
|
|
t.Helper()
|
|
entries, _ := os.ReadDir(filepath.Join(setAsideRoot(r.drive), "tandoor"))
|
|
var out []string
|
|
for _, e := range entries {
|
|
out = append(out, filepath.Join(setAsideRoot(r.drive), "tandoor", e.Name()))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The sweep's measured order. Today (`e1613ae`) the refresh rewrites the kept unit's app.yaml with install B's
|
|
// password beside install A's data. After D47 the kept unit is set aside byte for byte, it is listed, and the
|
|
// primary place holds install B's own unit with none of A's data.
|
|
func TestD47_TheSweepOrderNeverRewritesTheKeptUnit(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
before := treeFingerprint(t, r.unitDir())
|
|
r.reinstall("bbbb000000000002")
|
|
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // the five-minute refresh
|
|
t.Fatalf("refresh: %v", err)
|
|
}
|
|
|
|
aside := r.setAside(t)
|
|
if len(aside) != 1 {
|
|
t.Fatalf("install A's unit must be set aside whole (one folder), found %v; primary app.yaml now: %q",
|
|
aside, mustRead(t, filepath.Join(RecoveryUnitComposePath(r.drive, "tandoor"), "app.yaml")))
|
|
}
|
|
if after := treeFingerprint(t, aside[0]); after != before {
|
|
t.Fatalf("the set-aside unit must be install A's unit byte for byte\nbefore:\n%s\nafter:\n%s", before, after)
|
|
}
|
|
primaryAppYaml := mustRead(t, filepath.Join(RecoveryUnitComposePath(r.drive, "tandoor"), "app.yaml"))
|
|
if strings.Contains(primaryAppYaml, d47PwA) || !strings.Contains(primaryAppYaml, "bbbb000000000002") {
|
|
t.Fatalf("the primary place must hold install B's own unit: %q", primaryAppYaml)
|
|
}
|
|
if fileExists(filepath.Join(AppDBDumpPath(r.drive, "tandoor"), "tandoor-postgres.sql")) {
|
|
t.Fatal("install A's database copy must not stay under install B's unit")
|
|
}
|
|
var rec *settings.KeptCopy
|
|
for _, k := range r.sett.GetKeptCopies() {
|
|
if k.Path == aside[0] {
|
|
kk := k
|
|
rec = &kk
|
|
}
|
|
}
|
|
if rec == nil || rec.Kind != KeptKindUnit || rec.Reason != KeptReasonEarlierInstall || rec.Source != "tandoor" {
|
|
t.Fatalf("the set-aside unit must be recorded as a kept copy (listed, removable only by the household): %+v", r.sett.GetKeptCopies())
|
|
}
|
|
}
|
|
|
|
// A unit written BEFORE D47 records no install: the carried database password decides (a reinstall mints a new one).
|
|
func TestD47_AUnitFromBeforeD47IsSetAsideWhenItsPasswordDiffers(t *testing.T) {
|
|
r := d47Setup(t, "")
|
|
before := treeFingerprint(t, r.unitDir())
|
|
r.reinstall("")
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil {
|
|
t.Fatalf("refresh: %v", err)
|
|
}
|
|
aside := r.setAside(t)
|
|
if len(aside) != 1 || treeFingerprint(t, aside[0]) != before {
|
|
t.Fatalf("an unrecorded unit with another install's password must be set aside whole, found %v", aside)
|
|
}
|
|
}
|
|
|
|
// The set-aside unit restores with ITS OWN password (the household's „Visszaállítás" on it), never install B's.
|
|
func TestD47_TheSetAsideUnitRestoresWithItsOwnPassword(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
r.reinstall("bbbb000000000002")
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ks := r.sett.GetKeptCopies()
|
|
if len(ks) != 1 {
|
|
t.Fatalf("one kept copy expected, got %+v", ks)
|
|
}
|
|
r.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
|
|
return []DiscoveredDB{{StackName: "tandoor", ContainerName: "tandoor-postgres", DBType: DBTypePostgres}}, nil
|
|
}
|
|
var imported []string
|
|
r.m.importDBDump = func(_ context.Context, _ DiscoveredDB, p string) error { imported = append(imported, p); return nil }
|
|
r.m.volumeReplayFrom = func(string, string) (int, error) { return 1, nil }
|
|
r.m.undoVolumeProbeFn = func(string) (int64, bool, error) { return 10, true, nil }
|
|
r.m.undoVolumeTarFn = func(vol, dir string) error {
|
|
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("live"), 0o600)
|
|
}
|
|
|
|
app, res, err := r.m.RestoreSetAsideUnit(ks[0].ID)
|
|
if err != nil || app != "tandoor" {
|
|
t.Fatalf("restore of the set-aside unit: app=%q err=%v", app, err)
|
|
}
|
|
if got := r.prov.gotEnv["DB_PASSWORD"]; got != d47PwA {
|
|
t.Fatalf("the restored app must get install A's password (the one its data opens with), got %q", got)
|
|
}
|
|
if res.DBsReplayed != 1 || len(imported) != 1 || !strings.Contains(imported[0], filepath.Join(setAsideDirName, "tandoor")) {
|
|
t.Fatalf("the database copy must come from the set-aside unit: %v (replayed %d)", imported, res.DBsReplayed)
|
|
}
|
|
}
|
|
|
|
// The guard is in the WRITER: a set-aside that cannot be done refuses the write, the unit stays byte for byte, and the
|
|
// ERROR line names the app and both identities.
|
|
func TestD47_TheWriterRefusesWhenTheUnitCannotBeSetAside(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
before := treeFingerprint(t, r.unitDir())
|
|
r.reinstall("bbbb000000000002")
|
|
renameUnitFn = func(string, string) error { return errors.New("injected: rename refused") }
|
|
defer func() { renameUnitFn = os.Rename }()
|
|
|
|
err := r.m.captureRecoveryUnit("tandoor", false)
|
|
if !errors.Is(err, errUnitOfAnotherInstall) {
|
|
t.Fatalf("the capture must refuse, got %v", err)
|
|
}
|
|
if after := treeFingerprint(t, r.unitDir()); after != before {
|
|
t.Fatalf("a refused write must leave install A's unit untouched\nbefore:\n%s\nafter:\n%s", before, after)
|
|
}
|
|
line := r.logs.String()
|
|
if !strings.Contains(line, "[ERROR]") || !strings.Contains(line, "tandoor") || !strings.Contains(line, "aaaa000000000001") || !strings.Contains(line, "bbbb000000000002") {
|
|
t.Fatalf("the ERROR line must name the app and both identities: %s", line)
|
|
}
|
|
if strings.Contains(line, d47PwA) || strings.Contains(line, d47PwB) {
|
|
t.Fatal("SECRET LEAK: a password value reached the log")
|
|
}
|
|
}
|
|
|
|
// The data legs ask too: install B's night database dump never lands in install A's unit.
|
|
func TestD47_TheNightDatabaseDumpSetsTheEarlierUnitAsideFirst(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
before := treeFingerprint(t, r.unitDir())
|
|
r.reinstall("bbbb000000000002")
|
|
r.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
|
|
return []DiscoveredDB{{StackName: "tandoor", ContainerName: "tandoor-postgres", DBType: DBTypePostgres}}, nil
|
|
}
|
|
r.m.dumpOne = func(_ context.Context, db DiscoveredDB, dir string, _ *log.Logger, _ bool) DumpResult {
|
|
p := filepath.Join(dir, "tandoor-postgres.sql")
|
|
_ = os.MkdirAll(dir, 0o755)
|
|
_ = os.WriteFile(p, []byte(pgDump(1)+"-- install B, empty\n"), 0o644)
|
|
return DumpResult{DB: db, FilePath: p, Size: 10}
|
|
}
|
|
if err := r.m.guardUnitWrite("tandoor", r.drive, "database dump"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res := r.m.dumpOneOrDefault(context.Background(), r.m.mustDB(t), AppDBDumpPath(r.drive, "tandoor"))
|
|
if res.Error != nil {
|
|
t.Fatal(res.Error)
|
|
}
|
|
aside := r.setAside(t)
|
|
if len(aside) != 1 || treeFingerprint(t, aside[0]) != before {
|
|
t.Fatalf("install A's unit must be set aside whole before install B's dump is written, found %v", aside)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) mustDB(t *testing.T) DiscoveredDB {
|
|
t.Helper()
|
|
dbs, err := m.discoverDBs(context.Background())
|
|
if err != nil || len(dbs) == 0 {
|
|
t.Fatal("no db")
|
|
}
|
|
return dbs[0]
|
|
}
|
|
|
|
// The other side: a normal backup of the SAME install still refreshes the unit whole, and nothing is set aside.
|
|
func TestD47_ASameInstallBackupStillRefreshesTheUnit(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
mustWrite(t, filepath.Join(AppDBDumpPath(r.drive, "tandoor"), "tandoor-postgres.sql"), pgDump(1)+"-- tonight's rows\n")
|
|
if err := r.m.CaptureRecoveryUnit("tandoor"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if aside := r.setAside(t); len(aside) != 0 {
|
|
t.Fatalf("a backup of the same install must not set anything aside: %v", aside)
|
|
}
|
|
appy := mustRead(t, filepath.Join(RecoveryUnitComposePath(r.drive, "tandoor"), "app.yaml"))
|
|
if !strings.Contains(appy, d47PwA) || !strings.Contains(appy, "install_id: aaaa000000000001") {
|
|
t.Fatalf("the same install's unit carries its own password and identity: %q", appy)
|
|
}
|
|
if got := mustRead(t, filepath.Join(AppDBDumpPath(r.drive, "tandoor"), "tandoor-postgres.sql")); !strings.Contains(got, "tonight's rows") {
|
|
t.Fatal("tonight's dump must be in the unit")
|
|
}
|
|
}
|
|
|
|
// A restore of the app runs: the refresh never moves the unit it reads (refused, untouched).
|
|
func TestD47_NoSetAsideWhileTheAppIsRestored(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
before := treeFingerprint(t, r.unitDir())
|
|
r.reinstall("bbbb000000000002")
|
|
r.m.BeginRestoreOp("restore", "tandoor")
|
|
if err := r.m.guardUnitWrite("tandoor", r.drive, "database dump"); !errors.Is(err, errUnitOfAnotherInstall) {
|
|
t.Fatalf("a write during the app's restore must refuse, got %v", err)
|
|
}
|
|
if after := treeFingerprint(t, r.unitDir()); after != before {
|
|
t.Fatal("the unit under a restore must stay as it is")
|
|
}
|
|
}
|
|
|
|
// The verdict as a pure function: ids decide when both are known; otherwise a differing carried secret.
|
|
func TestD47_OwnerVerdict(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
unitID, id string
|
|
us, ls map[string]string
|
|
want unitOwner
|
|
}{
|
|
{"same id", "a", "a", nil, nil, ownerSame},
|
|
{"other id", "a", "b", map[string]string{"P": "x"}, map[string]string{"P": "x"}, ownerOther},
|
|
{"unrecorded unit, same password", "", "b", map[string]string{"P": "x"}, map[string]string{"P": "x"}, ownerSame},
|
|
{"unrecorded unit, other password", "", "b", map[string]string{"P": "x"}, map[string]string{"P": "y"}, ownerOther},
|
|
{"unrecorded unit, no secrets", "", "b", nil, nil, ownerSame},
|
|
{"empty value never decides", "", "", map[string]string{"P": ""}, map[string]string{"P": "y"}, ownerSame},
|
|
}
|
|
for _, c := range cases {
|
|
if got, _ := unitOwnerVerdict(c.unitID, c.id, c.us, c.ls); got != c.want {
|
|
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// D47 follow-up (brief 2026-10-26 Part 5.2): a set-aside tells the operator once — the box's own households must not
|
|
// set units aside silently after the floor. COMPANION RED-PROOF: drop the notify call → this fails.
|
|
func TestD47_ASetAsideTellsTheOperatorOnce(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
var got []string
|
|
r.m.SetUnitSetAsideNotify(func(stack, unitID, liveID, dest string) { got = append(got, stack+" "+unitID+" "+liveID) })
|
|
r.reinstall("bbbb000000000002")
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // the next refresh: nothing more to set aside
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0] != "tandoor aaaa000000000001 bbbb000000000002" {
|
|
t.Fatalf("the operator hears once, with both identities: %v", got)
|
|
}
|
|
}
|
|
|
|
// R-1288, the Tester 1 order measured 2026-10-25 on 0.343.0: install A backed up → reinstall B → B's first backup sets
|
|
// A's unit aside (one alarm) and writes B's own unit → the household restores A's set-aside copy. Before: the next
|
|
// backup found B's unit and set it aside with a SECOND alarm. Now B's unit is set aside AT the restore, quietly, and
|
|
// the next backup writes A's unit with nothing more set aside.
|
|
// RED-PROOF: drop the retireReplacedUnit call → after the restore the primary place still holds B's unit, and the
|
|
// next capture alarms a second time.
|
|
func TestR1288_ARestoreOfASetAsideUnitRetiresTheReplacedUnitQuietly(t *testing.T) {
|
|
r := d47Setup(t, "aaaa000000000001")
|
|
var alarms []string
|
|
r.m.SetUnitSetAsideNotify(func(stack, unitID, liveID, dest string) { alarms = append(alarms, unitID) })
|
|
r.reinstall("bbbb000000000002")
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // B's first backup: A set aside, B's unit written
|
|
t.Fatal(err)
|
|
}
|
|
if len(alarms) != 1 || len(r.setAside(t)) != 1 {
|
|
t.Fatalf("precondition: one set-aside, one alarm; got alarms %v, set-aside %v", alarms, r.setAside(t))
|
|
}
|
|
r.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
|
|
return []DiscoveredDB{{StackName: "tandoor", ContainerName: "tandoor-postgres", DBType: DBTypePostgres}}, nil
|
|
}
|
|
r.m.importDBDump = func(context.Context, DiscoveredDB, string) error { return nil }
|
|
r.m.volumeReplayFrom = func(string, string) (int, error) { return 1, nil }
|
|
r.m.undoVolumeProbeFn = func(string) (int64, bool, error) { return 10, true, nil }
|
|
r.m.undoVolumeTarFn = func(vol, dir string) error {
|
|
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("live"), 0o600)
|
|
}
|
|
ks := r.sett.GetKeptCopies()
|
|
if _, _, err := r.m.RestoreSetAsideUnit(ks[0].ID); err != nil {
|
|
t.Fatalf("restore of A's set-aside unit: %v", err)
|
|
}
|
|
if r.prov.info.InstallID != "aaaa000000000001" {
|
|
t.Fatalf("precondition: the restored app is install A, got %q", r.prov.info.InstallID)
|
|
}
|
|
if exists, id, _ := unitOwnerOf(r.unitDir()); exists {
|
|
t.Fatalf("after the restore the primary place still holds install %q's unit — it must be set aside at the restore", id)
|
|
}
|
|
if got := len(r.setAside(t)); got != 2 {
|
|
t.Fatalf("set-aside units after the restore: %d, want 2 (A's, kept as the household's copy, and B's)", got)
|
|
}
|
|
if len(alarms) != 1 {
|
|
t.Fatalf("the restore the household chose must not alarm: alarms %v", alarms)
|
|
}
|
|
if !strings.Contains(r.logs.String(), "replaced by the restored install") {
|
|
t.Fatal("the restore's set-aside must say why in one line")
|
|
}
|
|
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // the next backup: A's own unit, nothing set aside
|
|
t.Fatal(err)
|
|
}
|
|
if len(alarms) != 1 || len(r.setAside(t)) != 2 {
|
|
t.Fatalf("the next backup set something aside again: alarms %v, set-aside %v", alarms, r.setAside(t))
|
|
}
|
|
}
|