diff --git a/controller/internal/appbackup/dbdump.go b/controller/internal/appbackup/dbdump.go index 015a513..3d3b782 100644 --- a/controller/internal/appbackup/dbdump.go +++ b/controller/internal/appbackup/dbdump.go @@ -432,7 +432,12 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation { } // ListDumpFiles returns info about SQL dump files on disk. -func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) { +// +// M18: ValidateDump scans the dump line-by-line; on a customer with hundreds-of-MB dumps that is wasted +// disk I/O + CPU on every ~5-min scheduler cycle. `cached` is an optional lookup that returns a +// previously-computed DumpValidation for a file whose (name, size, modtime) match — when it returns ok, +// the expensive ValidateDump is skipped. Pass nil to always validate (legacy fast path / other callers). +func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) { entries, err := os.ReadDir(dumpDir) if err != nil { if os.IsNotExist(err) { @@ -474,7 +479,14 @@ func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) { f.StackName = base } - // Run validation on the file + // M18: reuse a cached validation when the file is unchanged (name+size+modtime), else validate. + if cached != nil { + if v, ok := cached(e.Name(), info.Size(), info.ModTime()); ok { + f.Validation = v + files = append(files, f) + continue + } + } fullPath := filepath.Join(dumpDir, e.Name()) f.Validation = ValidateDump(fullPath, f.DBType) diff --git a/controller/internal/appbackup/listdumpfiles_cache_test.go b/controller/internal/appbackup/listdumpfiles_cache_test.go new file mode 100644 index 0000000..73e00a9 --- /dev/null +++ b/controller/internal/appbackup/listdumpfiles_cache_test.go @@ -0,0 +1,94 @@ +package appbackup + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// writeDumpFile creates a minimal VALID postgres dump so a real ValidateDump would set Valid=true and +// TableCount=1 — distinguishable from the sentinel the cache returns. +func writeDumpFile(t *testing.T, dir string) string { + t.Helper() + path := filepath.Join(dir, "romm-postgres.sql") + body := "-- PostgreSQL database dump\n" + + "CREATE TABLE public.t (id int);\n" + + "-- PostgreSQL database dump complete\n" + + // pad past the 100-byte floor + "-- padding ------------------------------------------------------------\n" + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// TestListDumpFiles_CacheHitSkipsValidate asserts M18: when the `cached` lookup returns ok for an +// unchanged file, ListDumpFiles reuses that result and does NOT re-run the (expensive) ValidateDump. +// Proven via a sentinel TableCount=999 that only the cache could supply (a real validate yields 1). +func TestListDumpFiles_CacheHitSkipsValidate(t *testing.T) { + dir := t.TempDir() + writeDumpFile(t, dir) + + const sentinel = 999 + calls := 0 + cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) { + calls++ + return DumpValidation{Valid: true, TableCount: sentinel}, true // always "hit" + } + + files, err := ListDumpFiles(dir, cached) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 { + t.Fatalf("expected 1 dump file, got %d", len(files)) + } + if files[0].Validation.TableCount != sentinel { + t.Fatalf("validation TableCount = %d, want %d (sentinel from cache) — ValidateDump was re-run instead of using the cache", + files[0].Validation.TableCount, sentinel) + } + if calls != 1 { + t.Fatalf("cached lookup called %d times, want 1", calls) + } +} + +// TestListDumpFiles_CacheMissValidates asserts that on a cache MISS (e.g. modtime changed) ListDumpFiles +// falls through to a real ValidateDump (sentinel must NOT appear; real TableCount=1). +func TestListDumpFiles_CacheMissValidates(t *testing.T) { + dir := t.TempDir() + writeDumpFile(t, dir) + + cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) { + return DumpValidation{Valid: true, TableCount: 999}, false // always "miss" + } + + files, err := ListDumpFiles(dir, cached) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 { + t.Fatalf("expected 1 dump file, got %d", len(files)) + } + if files[0].Validation.TableCount == 999 { + t.Fatalf("got the sentinel on a cache MISS — should have run a real ValidateDump") + } + if !files[0].Validation.Valid || files[0].Validation.TableCount != 1 { + t.Fatalf("real validation expected Valid=true TableCount=1, got Valid=%v TableCount=%d", + files[0].Validation.Valid, files[0].Validation.TableCount) + } +} + +// TestListDumpFiles_NilCachedAlwaysValidates asserts the legacy fast path (cached==nil) still validates. +func TestListDumpFiles_NilCachedAlwaysValidates(t *testing.T) { + dir := t.TempDir() + writeDumpFile(t, dir) + + files, err := ListDumpFiles(dir, nil) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || !files[0].Validation.Valid || files[0].Validation.TableCount != 1 { + t.Fatalf("nil-cached path should validate: got %+v", files[0].Validation) + } +} diff --git a/controller/internal/backup/appbackup_bridge.go b/controller/internal/backup/appbackup_bridge.go index 45bc35f..5399d4e 100644 --- a/controller/internal/backup/appbackup_bridge.go +++ b/controller/internal/backup/appbackup_bridge.go @@ -14,6 +14,7 @@ package backup import ( "context" "log" + "time" "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" ) @@ -68,8 +69,8 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation { return appbackup.ValidateDump(filePath, dbType) } -func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) { - return appbackup.ListDumpFiles(dumpDir) +func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) { + return appbackup.ListDumpFiles(dumpDir, cached) } // --- function forwarders (appdata) --- diff --git a/controller/internal/backup/backup.go b/controller/internal/backup/backup.go index d266818..6c0b8b1 100644 --- a/controller/internal/backup/backup.go +++ b/controller/internal/backup/backup.go @@ -233,6 +233,8 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error { ValidatedAt: time.Now().Format(time.RFC3339), TableCount: result.Validation.TableCount, HasHeader: result.Validation.Valid, + Size: result.Validation.FileSize, + ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339), } if !result.Validation.Valid { cache.Error = result.Validation.Error @@ -467,6 +469,8 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error { ValidatedAt: time.Now().Format(time.RFC3339), TableCount: result.Validation.TableCount, HasHeader: result.Validation.Valid, + Size: result.Validation.FileSize, + ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339), } if !result.Validation.Valid { cache.Error = result.Validation.Error @@ -478,13 +482,50 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error { } // listAllDumpFiles scans per-drive per-stack DB dump directories. +// +// M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is +// passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min +// cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime — +// so the write-back (and its settings.json disk write) only happens when a dump actually changed. func (m *Manager) listAllDumpFiles() []DumpFileInfo { + modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) } + + var cacheSnapshot map[string]settings.DBValidationCache + if m.settings != nil { + cacheSnapshot = m.settings.GetDBValidations() + } + lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) { + c, ok := cacheSnapshot[name] + if !ok || c.Size != size || c.ModTime != modKey(mod) { + return DumpValidation{}, false // cache miss / changed → validate + } + return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true + } + var allFiles []DumpFileInfo for drive, stacks := range m.groupStacksByDrive() { for _, stack := range stacks { dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name) - if files, err := ListDumpFiles(dumpDir); err == nil { - allFiles = append(allFiles, files...) + files, err := ListDumpFiles(dumpDir, lookup) + if err != nil { + continue + } + for _, f := range files { + // Write back only fresh validations (cache miss against the snapshot), so unchanged + // dumps cause neither a re-validation nor a settings.json write each cycle. + if m.settings != nil { + if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) { + _ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{ + ValidatedAt: time.Now().Format(time.RFC3339), + TableCount: f.Validation.TableCount, + HasHeader: f.Validation.Valid, + Error: f.Validation.Error, + Size: f.Size, + ModTime: modKey(f.ModTime), + }) + } + } + allFiles = append(allFiles, f) } } } diff --git a/controller/internal/settings/settings.go b/controller/internal/settings/settings.go index aaf0c06..e50e7fb 100644 --- a/controller/internal/settings/settings.go +++ b/controller/internal/settings/settings.go @@ -161,6 +161,11 @@ type DBValidationCache struct { TableCount int `json:"table_count"` HasHeader bool `json:"has_header"` Error string `json:"error,omitempty"` + // M18: Size + ModTime let ListDumpFiles skip the expensive line-by-line re-validation on every + // ~5-min scheduler cycle when the dump file is unchanged. A cache entry is a HIT only when both the + // file size and (RFC3339, second-precision) modtime match the on-disk file. + Size int64 `json:"size,omitempty"` + ModTime string `json:"mod_time,omitempty"` // RFC3339 (UTC) } // SetDebug enables or disables debug logging for settings operations.