M18: cache DB-dump validation; skip re-validate on unchanged dumps
ListDumpFiles ran ValidateDump (line-by-line scan) for every dump on every ~5-min RefreshCache cycle — wasted I/O+CPU on large customer dumps. ListDumpFiles now takes an optional cached(name,size,mod) lookup; on a (size+modtime) match it reuses the prior result and skips ValidateDump. settings.DBValidationCache gains Size+ModTime; listAllDumpFiles builds the lookup from the persisted cache and writes back only fresh validations (cache miss), so an unchanged dump triggers neither a re-validation nor a settings.json write each cycle. nil cached = legacy validate-always (back-compat). Tests: cache-hit skips validate (sentinel), cache-miss validates, nil validates.
This commit is contained in:
@@ -432,7 +432,12 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
|
||||
}
|
||||
|
||||
// ListDumpFiles returns info about SQL dump files on disk.
|
||||
func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
|
||||
//
|
||||
// M18: ValidateDump scans the dump line-by-line; on a customer with hundreds-of-MB dumps that is wasted
|
||||
// disk I/O + CPU on every ~5-min scheduler cycle. `cached` is an optional lookup that returns a
|
||||
// previously-computed DumpValidation for a file whose (name, size, modtime) match — when it returns ok,
|
||||
// the expensive ValidateDump is skipped. Pass nil to always validate (legacy fast path / other callers).
|
||||
func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
|
||||
entries, err := os.ReadDir(dumpDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -474,7 +479,14 @@ func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
|
||||
f.StackName = base
|
||||
}
|
||||
|
||||
// Run validation on the file
|
||||
// M18: reuse a cached validation when the file is unchanged (name+size+modtime), else validate.
|
||||
if cached != nil {
|
||||
if v, ok := cached(e.Name(), info.Size(), info.ModTime()); ok {
|
||||
f.Validation = v
|
||||
files = append(files, f)
|
||||
continue
|
||||
}
|
||||
}
|
||||
fullPath := filepath.Join(dumpDir, e.Name())
|
||||
f.Validation = ValidateDump(fullPath, f.DBType)
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package appbackup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// writeDumpFile creates a minimal VALID postgres dump so a real ValidateDump would set Valid=true and
|
||||
// TableCount=1 — distinguishable from the sentinel the cache returns.
|
||||
func writeDumpFile(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, "romm-postgres.sql")
|
||||
body := "-- PostgreSQL database dump\n" +
|
||||
"CREATE TABLE public.t (id int);\n" +
|
||||
"-- PostgreSQL database dump complete\n" +
|
||||
// pad past the 100-byte floor
|
||||
"-- padding ------------------------------------------------------------\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// TestListDumpFiles_CacheHitSkipsValidate asserts M18: when the `cached` lookup returns ok for an
|
||||
// unchanged file, ListDumpFiles reuses that result and does NOT re-run the (expensive) ValidateDump.
|
||||
// Proven via a sentinel TableCount=999 that only the cache could supply (a real validate yields 1).
|
||||
func TestListDumpFiles_CacheHitSkipsValidate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDumpFile(t, dir)
|
||||
|
||||
const sentinel = 999
|
||||
calls := 0
|
||||
cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
|
||||
calls++
|
||||
return DumpValidation{Valid: true, TableCount: sentinel}, true // always "hit"
|
||||
}
|
||||
|
||||
files, err := ListDumpFiles(dir, cached)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("expected 1 dump file, got %d", len(files))
|
||||
}
|
||||
if files[0].Validation.TableCount != sentinel {
|
||||
t.Fatalf("validation TableCount = %d, want %d (sentinel from cache) — ValidateDump was re-run instead of using the cache",
|
||||
files[0].Validation.TableCount, sentinel)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("cached lookup called %d times, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListDumpFiles_CacheMissValidates asserts that on a cache MISS (e.g. modtime changed) ListDumpFiles
|
||||
// falls through to a real ValidateDump (sentinel must NOT appear; real TableCount=1).
|
||||
func TestListDumpFiles_CacheMissValidates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDumpFile(t, dir)
|
||||
|
||||
cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
|
||||
return DumpValidation{Valid: true, TableCount: 999}, false // always "miss"
|
||||
}
|
||||
|
||||
files, err := ListDumpFiles(dir, cached)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("expected 1 dump file, got %d", len(files))
|
||||
}
|
||||
if files[0].Validation.TableCount == 999 {
|
||||
t.Fatalf("got the sentinel on a cache MISS — should have run a real ValidateDump")
|
||||
}
|
||||
if !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
|
||||
t.Fatalf("real validation expected Valid=true TableCount=1, got Valid=%v TableCount=%d",
|
||||
files[0].Validation.Valid, files[0].Validation.TableCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListDumpFiles_NilCachedAlwaysValidates asserts the legacy fast path (cached==nil) still validates.
|
||||
func TestListDumpFiles_NilCachedAlwaysValidates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDumpFile(t, dir)
|
||||
|
||||
files, err := ListDumpFiles(dir, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 1 || !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
|
||||
t.Fatalf("nil-cached path should validate: got %+v", files[0].Validation)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user