package stacks import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "sync" "testing" "time" ) // v0.273.0 — the PostgreSQL major conversion (`09` §6.4 part 10, pgconvert.go). Every test runs the REAL // guarded update job with the process boundaries faked (compose, the volume copier, the PostgreSQL // converter, the health waits) and reads the EFFECT back: the bytes in the fake database volume, the pin, // the phase, the hold, the kept copy. Nothing here reaches Docker (R-650). const ( convOld = "services:\n web:\n image: docmost/docmost:0.96.0\n db:\n image: postgres:16-alpine\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n" convNew = "services:\n web:\n image: docmost/docmost:0.96.0\n db:\n image: postgres:18-alpine\n volumes:\n - pgdata:/var/lib/postgresql\nvolumes:\n pgdata:\n" convVol = "nextcloud_pgdata" // project = the stack dir's name (newPinManager names it nextcloud) convPin = "deployed: true\nenv: {}\npinned_images:\n web: docmost/docmost:0.96.0\n db: postgres:16-alpine\n" convOldData = "PG16-DATADIR" convNewData = "PG18-DATADIR" ) func convLadder(mark string) string { e := ` - {"from": {"web": "docmost/docmost:0.96.0", "db": "postgres:16-alpine"}, "to": {"web": "docmost/docmost:0.96.0", "db": "postgres:18-alpine"}, "digest": {}, "verdict": "proven"` if mark != "" { e += `, "engine_conversion": ` + mark } return "update_ladder:\n" + e + "}\n" } const goodMark = `{"service": "db", "engine": "postgres", "from": 16, "to": 18}` // fakePG is the PostgreSQL boundary. It works on the fake copier's volume content, so "the data came // back" is a string compare. type fakePG struct { mu sync.Mutex fc *fakeCopier calls []string before []string after []string // nil = equal to before loadErr error dumpCut bool version string startErr error onConvert func(step string) // a hook to act between steps (the restart test) } func (p *fakePG) note(s string) { p.mu.Lock() p.calls = append(p.calls, s) p.mu.Unlock() if p.onConvert != nil { p.onConvert(s) } } func (p *fakePG) ServiceContainer(project, service string) (string, error) { return project + "-" + service, nil } func (p *fakePG) Start(c string) error { p.note("start " + c); return p.startErr } func (p *fakePG) Stop(c string) error { p.note("stop " + c); return nil } func (p *fakePG) Env(string, string) string { return "docmost" } func (p *fakePG) WaitReady(context.Context, string, string, time.Duration) error { return nil } func (p *fakePG) Snapshot(c, user string) ([]string, error) { if p.fc.vol(convVol) == convOldData { p.note("snapshot old") return p.before, nil } p.note("snapshot new") if p.after != nil { return p.after, nil } return p.before, nil } func (p *fakePG) DumpAll(c, user, path string) error { p.note("dumpall") body := "CREATE ROLE docmost;\nALTER ROLE docmost WITH SUPERUSER;\n-- data\n" if !p.dumpCut { body += "--\n-- " + dumpAllCompleteLine + "\n--\n" } return os.WriteFile(path, []byte(body), 0o600) } func (p *fakePG) Empty(copyVol, vol string) error { p.note("empty " + vol) if !p.fc.Complete(copyVol) { return fmt.Errorf("no marker in %s", copyVol) } p.fc.setVol(vol, "") return nil } func (p *fakePG) DropEmptyDatabases(string, string) ([]string, error) { p.note("drop-empty") return []string{"docmost"}, nil } func (p *fakePG) Roles(string, string) ([]string, error) { return []string{"docmost"}, nil } func (p *fakePG) Load(c, user, path string, skip map[string]bool) error { p.note("load") if !skip["CREATE ROLE docmost;"] { return fmt.Errorf("the existing role's CREATE line was not skipped") } if p.loadErr != nil { return p.loadErr } p.fc.setVol(convVol, convNewData) return nil } func (p *fakePG) DataVersion(string) (string, error) { if p.version != "" { return p.version, nil } return "18\n", nil } func (p *fakePG) called(prefix string) bool { p.mu.Lock() defer p.mu.Unlock() for _, c := range p.calls { if strings.HasPrefix(c, prefix) { return true } } return false } // convManager: slice 4's shape with docmost on postgres:16-alpine and the catalog offering 18 through a // one-entry ladder carrying `mark` ("" = no mark). The new version is healthy unless newHealthy is false; // the old one always answers its old probe. func convManager(t *testing.T, mark string) (*Manager, string, *fakeGuards, *composeRec, *fakeCopier, *fakePG) { t.Helper() m, dir := newPinManager(t, convOld, convNew, convPin) mustWrite(t, AppliedComposePath(dir), convOld) catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml")) mustWrite(t, filepath.Join(catDir, ".felhom.yml"), "display_name: Docmost\n"+convLadder(mark)) mustWrite(t, filepath.Join(dir, ".felhom.yml"), "display_name: Docmost\n") m.stacks["nextcloud"].Meta = Metadata{DisplayName: "Docmost"} g := &fakeGuards{points: []UpdateRestorePoint{{Tier: UpdateTierSecondDrive, ProvenAt: slice4T0.Add(-1 * time.Hour)}}, stackDir: dir} c := &composeRec{fail: map[string]error{}} m.updateGuards = g m.updateComposeFn = c.fn m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return true, "fake healthy" } m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) { return true, "old answered" } m.updateMemoryFn = func(int, int, int, int) (error, string) { return nil, "" } m.updateDiskFreeFn = func() (float64, bool) { return 50, true } m.convertFreeFn = func(string) (int64, bool) { return 50 << 30, true } m.updateNowFn = func() time.Time { return slice4T0 } m.execFn = func(string, ...string) (string, error) { return "", nil } fc := newFakeCopier(map[string]string{convVol: convOldData}) m.undoCopier = fc pg := &fakePG{fc: fc, before: []string{"db:docmost owner=docmost", "role:docmost super=true", "ext:docmost:pg_trgm", "rows:docmost:public.users=1"}} m.pgConv = pg return m, dir, g, c, fc, pg } func dbPin(t *testing.T, dir string) string { t.Helper() return LoadAppConfig(dir).PinnedImages["db"] } // TestConvert_HappyPath — one press converts 16 → 18: the volume holds the new datadir, the pin names 18, // the phase passed through `converting`, the OLD datadir's copy is KEPT and recorded, the dump is gone. func TestConvert_HappyPath(t *testing.T) { m, dir, _, c, fc, pg := convManager(t, goodMark) if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } st := waitUpdateDone(t, m, "nextcloud") if st.UpdatePhase != UpdatePhaseDone { t.Fatalf("ended %q (%s)", st.UpdatePhase, st.UpdateError) } if got := fc.vol(convVol); got != convNewData { t.Fatalf("the database volume holds %q, want the converted datadir", got) } if got := dbPin(t, dir); got != "postgres:18-alpine" { t.Fatalf("pin %q, want postgres:18-alpine", got) } order := strings.Join(pg.calls, ",") for _, want := range []string{"start nextcloud-db", "snapshot old", "dumpall", "stop nextcloud-db", "empty " + convVol, "drop-empty", "load", "snapshot new"} { if !strings.Contains(order, want) { t.Fatalf("the conversion never did %q: %s", want, order) } } if strings.Index(order, "dumpall") > strings.Index(order, "empty") { t.Fatalf("the volume was emptied before the dump was taken: %s", order) } if !strings.Contains(strings.Join(c.list(), ","), "up -d --no-deps db") { t.Fatalf("the new engine was never started alone: %v", c.list()) } cc := LoadAppConfig(dir).ConversionCopy if cc == nil || cc.Volume != convVol || cc.From != 16 || cc.To != 18 { t.Fatalf("the kept copy is not recorded: %+v", cc) } if fc.nCopies() != 1 || fc.copies[cc.Copy] != convOldData { t.Fatalf("the OLD datadir's copy must be kept after success (B5); copies=%v", fc.copies) } if _, err := os.Stat(filepath.Join(dir, preUpdateConvertDir)); !os.IsNotExist(err) { t.Fatal("the conversion's dump must be removed after success") } } // TestConvert_MajorWithoutMarkIsRefused — B1's defence in depth: a step moving PostgreSQL across a major // with no engine_conversion mark is refused BEFORE anything moves, with the household sentence. // // COMPANION RED-PROOF (REPORT.md): at v0.272.0 there is no such check — the update pins 18 and runs; // reproduced by making planEngineConversion return nil, nil: this test fails at "the preflight let it through". func TestConvert_MajorWithoutMarkIsRefused(t *testing.T) { m, dir, _, c, fc, pg := convManager(t, "") ref := m.UpdatePreflight("nextcloud") if ref == nil || ref.Reason != "engine_no_test" { t.Fatalf("the preflight let it through: %+v", ref) } if want := "Ez a lépés a(z) Docmost adatbázisának fő verzióját váltaná, de nincs róla próba. Nem változott semmi."; ref.Error() != want { t.Fatalf("sentence %q, want %q", ref.Error(), want) } if err := m.StartGuardedUpdate("nextcloud"); err == nil { t.Fatal("StartGuardedUpdate accepted a PostgreSQL major move with no test") } if dbPin(t, dir) != "postgres:16-alpine" || fc.vol(convVol) != convOldData || len(c.list()) != 0 || len(pg.calls) != 0 { t.Fatalf("something moved: pin=%s vol=%s compose=%v pg=%v", dbPin(t, dir), fc.vol(convVol), c.list(), pg.calls) } } // TestConvert_JobRefusesAMarkThatDoesNotMatch — the job itself (not only the preflight) refuses a mark // naming a different major; nothing is pulled. func TestConvert_JobRefusesAMarkThatDoesNotMatch(t *testing.T) { m, dir, _, c, _, _ := convManager(t, `{"service": "db", "engine": "postgres", "from": 16, "to": 17}`) m.runGuardedUpdate(context.Background(), "nextcloud") st, _ := m.GetStack("nextcloud") if st.UpdatePhase != UpdatePhaseFailed || st.UpdateErrorKey != "err.stacks.update_engine_no_test" { t.Fatalf("phase %q key %q, want failed with the no-test sentence", st.UpdatePhase, st.UpdateErrorKey) } if dbPin(t, dir) != "postgres:16-alpine" { t.Fatal("the pin moved") } for _, call := range c.list() { if strings.HasPrefix(call, "pull") { t.Fatal("pulled before refusing") } } } func assertUndoneOnOld(t *testing.T, m *Manager, dir string, fc *fakeCopier) { t.Helper() st := waitUpdateDone(t, m, "nextcloud") if st.UpdatePhase != UpdatePhaseUndone { t.Fatalf("ended %q (%s), want undone", st.UpdatePhase, st.UpdateError) } if got := fc.vol(convVol); got != convOldData { t.Fatalf("the database volume holds %q after the undo, want the OLD datadir back", got) } if got := dbPin(t, dir); got != "postgres:16-alpine" { t.Fatalf("pin %q after the undo, want 16", got) } if LoadAppConfig(dir).ConversionCopy != nil { t.Fatal("an undone conversion must not record a kept copy") } } // TestConvert_CountsDifferAreUndone — the check after the load differs → undo, the old datadir back. func TestConvert_CountsDifferAreUndone(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) pg.after = []string{"db:docmost owner=docmost", "role:docmost super=true", "ext:docmost:pg_trgm", "rows:docmost:public.users=0"} if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } assertUndoneOnOld(t, m, dir, fc) } // TestConvert_LoadFailureIsUndone — case (a) of Part D: the load errors → undo. func TestConvert_LoadFailureIsUndone(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) pg.loadErr = errors.New("ERROR: relation already exists") if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } assertUndoneOnOld(t, m, dir, fc) } // TestConvert_HealthFailureIsUndone — case (b): converted fine, the app unhealthy on the new engine → undo. func TestConvert_HealthFailureIsUndone(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "new version unhealthy" } if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } assertUndoneOnOld(t, m, dir, fc) if !pg.called("load") { t.Fatal("the fixture never reached the load — this test must fail AFTER the conversion") } } // TestConvert_WrongVersionIsUndone — the new datadir's PG_VERSION is not the mark's `to` → undo. func TestConvert_WrongVersionIsUndone(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) pg.version = "17" if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } assertUndoneOnOld(t, m, dir, fc) } // TestConvert_CutOffDumpNeverEmptiesTheVolume — a dump without its completion line stops the conversion // BEFORE the volume is touched. func TestConvert_CutOffDumpNeverEmptiesTheVolume(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) pg.dumpCut = true if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } assertUndoneOnOld(t, m, dir, fc) if pg.called("empty") { t.Fatal("the volume was emptied after a cut-off dump") } } // TestConvert_NeverEmptiedWithoutMarker — rule 3 of the brief: the DB volume is never emptied before the // undo copy's finished-marker is validated. The copy is made WITHOUT its marker; the volume must keep the // old datadir and the app must be held (the undo cannot use the copy either). // // COMPANION RED-PROOF (REPORT.md): drop the Complete() check before Empty in convertEngine AND the marker // test inside Empty — this test fails at "the volume was emptied". func TestConvert_NeverEmptiedWithoutMarker(t *testing.T) { m, _, g, _, fc, pg := convManager(t, goodMark) fc.cutOff = true if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } st := waitUpdateDone(t, m, "nextcloud") if fc.vol(convVol) != convOldData { t.Fatalf("the volume was emptied (holds %q) without a validated copy", fc.vol(convVol)) } if pg.called("empty") { t.Fatal("Empty was called although the copy had no marker") } if held, _ := g.HoldFor("nextcloud"); !held || st.UpdatePhase != UpdatePhaseFailed { t.Fatalf("held=%v phase=%q — with no usable copy the app must be HELD (data untouched)", held, st.UpdatePhase) } } // TestConvert_NoSpaceIsRefusedWithNothingMoved — B6/A5: too little room beside the stack dir → refused // before the pin, the pull or the copy, with the household sentence naming both sizes. func TestConvert_NoSpaceIsRefusedWithNothingMoved(t *testing.T) { m, dir, _, c, fc, pg := convManager(t, goodMark) m.convertFreeFn = func(string) (int64, bool) { return 1 << 30, true } m.runGuardedUpdate(context.Background(), "nextcloud") st, _ := m.GetStack("nextcloud") if st.UpdatePhase != UpdatePhaseFailed || st.UpdateErrorKey != "err.stacks.update_convert_space" { t.Fatalf("phase %q key %q", st.UpdatePhase, st.UpdateErrorKey) } if !strings.HasPrefix(st.UpdateError, "A(z) Docmost adatbázisának átalakításához ") || !strings.HasSuffix(st.UpdateError, "Nem változott semmi.") { t.Fatalf("sentence %q", st.UpdateError) } if dbPin(t, dir) != "postgres:16-alpine" || fc.nCopies() != 0 || len(pg.calls) != 0 { t.Fatalf("something moved: pin=%s copies=%d pg=%v", dbPin(t, dir), fc.nCopies(), pg.calls) } for _, call := range c.list() { if strings.HasPrefix(call, "pull") || strings.HasPrefix(call, "stop") { t.Fatalf("compose %q ran before the refusal", call) } } } // TestConvert_RestartDuringConvertingIsUndone — B4: the controller dies after the volume was emptied; the // journal says `converting`; the next start UNDOES it — the old datadir back, the old pin, `undone`. func TestConvert_RestartDuringConvertingIsUndone(t *testing.T) { m, dir, _, _, fc, pg := convManager(t, goodMark) killed := make(chan struct{}) var once sync.Once pg.onConvert = func(step string) { if strings.HasPrefix(step, "empty") { once.Do(func() { close(killed) }) select {} // the process "dies" here: this goroutine never returns } } go m.runGuardedUpdate(context.Background(), "nextcloud") select { case <-killed: case <-time.After(5 * time.Second): t.Fatal("never reached the empty step") } time.Sleep(50 * time.Millisecond) // let Empty's own write land fc.setVol(convVol, "") // what a real kill leaves: the volume emptied // a NEW manager process reading the same disk m2, _, _, _, _, _ := convManager(t, goodMark) m2.cfg, m2.stacks, m2.undoCopier = m.cfg, map[string]*Stack{"nextcloud": {Name: "nextcloud", Deployed: true, ComposePath: filepath.Join(dir, "docker-compose.yml"), AppConfig: LoadAppConfig(dir)}}, fc m2.pgConv = &fakePG{fc: fc} m2.updateGuards = m.updateGuards j := m2.readUpdateJournal().Updates["nextcloud"] if j.Phase != UpdatePhaseConverting || !j.ConvertTouched || !j.Copied { t.Fatalf("journal phase=%q touched=%v copied=%v — the restart must find `converting` with the copies", j.Phase, j.ConvertTouched, j.Copied) } if got := m2.RecoverUpdates(); len(got) != 1 { t.Fatalf("RecoverUpdates resumed %v", got) } m2.ResumeInterruptedUpdates(context.Background()) assertUndoneOnOld(t, m2, dir, fc) } // TestConvert_ReleaseAfterAProvenBackup — B5: the kept copy stays while no backup is proven after the // conversion, and goes (with its record) once one is. func TestConvert_ReleaseAfterAProvenBackup(t *testing.T) { m, dir, g, _, fc, _ := convManager(t, goodMark) if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseDone { t.Fatalf("setup: %q", st.UpdatePhase) } if err := m.ScanStacks(); err == nil { _ = err } m.mu.Lock() m.stacks["nextcloud"].AppConfig = LoadAppConfig(dir) m.mu.Unlock() if got := m.ReleaseConversionCopies(context.Background()); len(got) != 0 || fc.nCopies() != 1 { t.Fatalf("released %v with only a pre-conversion backup; copies=%d", got, fc.nCopies()) } g.mu.Lock() g.points = []UpdateRestorePoint{{Tier: UpdateTierLocal, ProvenAt: slice4T0.Add(time.Hour)}} g.mu.Unlock() if got := m.ReleaseConversionCopies(context.Background()); len(got) != 1 || fc.nCopies() != 0 { t.Fatalf("released %v after a proven backup; copies=%d", got, fc.nCopies()) } if LoadAppConfig(dir).ConversionCopy != nil { t.Fatal("the record must go with the copy") } } func TestConvert_PostgresMajorFromTags(t *testing.T) { for ref, want := range map[string]int{ "postgres:16-alpine": 16, "postgres:17.2": 17, "postgres:18-alpine@sha256:ab": 18, "postgis/postgis:16-3.5-alpine": 16, "pgvector/pgvector:pg16": 16, "ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0": 16, } { if got, ok := postgresMajor(ref); !ok || got != want { t.Errorf("%s → %d %v, want %d", ref, got, ok, want) } } if _, ok := postgresMajor("postgres"); ok { t.Error("an untagged ref has no major") } // within a major is not a conversion; a non-postgres move is not either if c, err := planEngineConversion(map[string]string{"db": "postgres:16-alpine"}, map[string]string{"db": "postgres:16.4-alpine"}, nil); c != nil || err != nil { t.Errorf("16 → 16.4 must be neither a conversion nor a refusal: %v %v", c, err) } if c, err := planEngineConversion(map[string]string{"r": "redis:7"}, map[string]string{"r": "redis:8"}, nil); c != nil || err != nil { t.Errorf("redis is not postgres: %v %v", c, err) } } // TestConvert_PostgresFamilyMatchesTheBackupSide — isPostgresImage must know every family the backup // side dumps as Postgres (appbackup.dbTypeForImage, R-484); read from its source, the only way across the // package boundary without exporting it. func TestConvert_PostgresFamilyMatchesTheBackupSide(t *testing.T) { src, err := os.ReadFile("../appbackup/dbservices.go") if err != nil { t.Fatal(err) } for _, fam := range []string{"postgres", "postgis", "pgvector", "timescaledb"} { if !strings.Contains(string(src), `strings.Contains(img, "`+fam+`")`) { t.Fatalf("the backup side no longer names %q — re-read it and update isPostgresImage", fam) } if !isPostgresImage("x/" + fam + ":1") { t.Errorf("isPostgresImage misses %q", fam) } } } func TestConvert_FilterSkipsOnlyExactLines(t *testing.T) { in := "CREATE ROLE docmost;\nCREATE ROLE docmost_ro;\nALTER ROLE docmost WITH PASSWORD 'x';\nCREATE ROLE docmost; \n" var out bytes.Buffer if err := filterLines(strings.NewReader(in), &out, map[string]bool{"CREATE ROLE docmost;": true}); err != nil { t.Fatal(err) } want := "CREATE ROLE docmost_ro;\nALTER ROLE docmost WITH PASSWORD 'x';\nCREATE ROLE docmost; \n" if out.String() != want { t.Fatalf("got %q", out.String()) } } func TestConvert_ValidateDumpAll(t *testing.T) { d := t.TempDir() good, cut := filepath.Join(d, "g"), filepath.Join(d, "c") mustWrite(t, good, strings.Repeat("x", 9000)+"\n-- "+dumpAllCompleteLine+"\n") mustWrite(t, cut, strings.Repeat("x", 9000)+"\n-- PostgreSQL database dump complete\n") if err := validateDumpAll(good); err != nil { t.Fatal(err) } if validateDumpAll(cut) == nil { t.Fatal("a dump ending with a PER-DATABASE completion line is not a whole cluster dump") } } // TestConvert_MarkDoesNotChangeOldLadderPrints — the new field is omitempty, so R-680's failed-step // records (tied to LadderPrint) survive the upgrade to v0.273.0. func TestConvert_MarkDoesNotChangeOldLadderPrints(t *testing.T) { b, _ := json.Marshal([]LadderEntry{{From: map[string]string{"a": "x:1"}, To: map[string]string{"a": "x:2"}, Verdict: "proven"}}) if strings.Contains(string(b), "engine_conversion") { t.Fatalf("an entry without the mark prints it: %s", b) } }