admin/felhom-controller
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M18: cache DB-dump validation; skip re-validate on unchanged dumps

Browse Source 
ListDumpFiles ran ValidateDump (line-by-line scan) for every dump on every ~5-min
RefreshCache cycle — wasted I/O+CPU on large customer dumps. ListDumpFiles now takes
an optional cached(name,size,mod) lookup; on a (size+modtime) match it reuses the
prior result and skips ValidateDump. settings.DBValidationCache gains Size+ModTime;
listAllDumpFiles builds the lookup from the persisted cache and writes back only fresh
validations (cache miss), so an unchanged dump triggers neither a re-validation nor a
settings.json write each cycle. nil cached = legacy validate-always (back-compat).
Tests: cache-hit skips validate (sentinel), cache-miss validates, nil validates.
This commit is contained in:
authentik Default Admin
2026-06-14 14:13:09 +02:00
parent 6bab68b132
commit f8afe5c055
5 changed files with 159 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controller/internal/appbackup/dbdump.go
+14 -2
View File
@@ -432,7 +432,12 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
}
// ListDumpFiles returns info about SQL dump files on disk.
func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
//
// M18: ValidateDump scans the dump line-by-line; on a customer with hundreds-of-MB dumps that is wasted
// disk I/O + CPU on every ~5-min scheduler cycle. `cached` is an optional lookup that returns a
// previously-computed DumpValidation for a file whose (name, size, modtime) match — when it returns ok,
// the expensive ValidateDump is skipped. Pass nil to always validate (legacy fast path / other callers).
func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
entries, err := os.ReadDir(dumpDir)
if err != nil {
if os.IsNotExist(err) {
@@ -474,7 +479,14 @@ func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
f.StackName = base
}
// Run validation on the file
// M18: reuse a cached validation when the file is unchanged (name+size+modtime), else validate.
if cached != nil {
if v, ok := cached(e.Name(), info.Size(), info.ModTime()); ok {
f.Validation = v
files = append(files, f)
continue
}
}
fullPath := filepath.Join(dumpDir, e.Name())
f.Validation = ValidateDump(fullPath, f.DBType)
controller/internal/appbackup/listdumpfiles_cache_test.go
+94
View File
@@ -0,0 +1,94 @@
package appbackup
import (
"os"
"path/filepath"
"testing"
"time"
)
// writeDumpFile creates a minimal VALID postgres dump so a real ValidateDump would set Valid=true and
// TableCount=1 — distinguishable from the sentinel the cache returns.
func writeDumpFile(t *testing.T, dir string) string {
t.Helper()
path := filepath.Join(dir, "romm-postgres.sql")
body := "-- PostgreSQL database dump\n" +
"CREATE TABLE public.t (id int);\n" +
"-- PostgreSQL database dump complete\n" +
// pad past the 100-byte floor
"-- padding ------------------------------------------------------------\n"
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
// TestListDumpFiles_CacheHitSkipsValidate asserts M18: when the `cached` lookup returns ok for an
// unchanged file, ListDumpFiles reuses that result and does NOT re-run the (expensive) ValidateDump.
// Proven via a sentinel TableCount=999 that only the cache could supply (a real validate yields 1).
func TestListDumpFiles_CacheHitSkipsValidate(t *testing.T) {
dir := t.TempDir()
writeDumpFile(t, dir)
const sentinel = 999
calls := 0
cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
calls++
return DumpValidation{Valid: true, TableCount: sentinel}, true // always "hit"
}
files, err := ListDumpFiles(dir, cached)
if err != nil {
t.Fatal(err)
}
if len(files) != 1 {
t.Fatalf("expected 1 dump file, got %d", len(files))
}
if files[0].Validation.TableCount != sentinel {
t.Fatalf("validation TableCount = %d, want %d (sentinel from cache) — ValidateDump was re-run instead of using the cache",
files[0].Validation.TableCount, sentinel)
}
if calls != 1 {
t.Fatalf("cached lookup called %d times, want 1", calls)
}
}
// TestListDumpFiles_CacheMissValidates asserts that on a cache MISS (e.g. modtime changed) ListDumpFiles
// falls through to a real ValidateDump (sentinel must NOT appear; real TableCount=1).
func TestListDumpFiles_CacheMissValidates(t *testing.T) {
dir := t.TempDir()
writeDumpFile(t, dir)
cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
return DumpValidation{Valid: true, TableCount: 999}, false // always "miss"
}
files, err := ListDumpFiles(dir, cached)
if err != nil {
t.Fatal(err)
}
if len(files) != 1 {
t.Fatalf("expected 1 dump file, got %d", len(files))
}
if files[0].Validation.TableCount == 999 {
t.Fatalf("got the sentinel on a cache MISS — should have run a real ValidateDump")
}
if !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
t.Fatalf("real validation expected Valid=true TableCount=1, got Valid=%v TableCount=%d",
files[0].Validation.Valid, files[0].Validation.TableCount)
}
}
// TestListDumpFiles_NilCachedAlwaysValidates asserts the legacy fast path (cached==nil) still validates.
func TestListDumpFiles_NilCachedAlwaysValidates(t *testing.T) {
dir := t.TempDir()
writeDumpFile(t, dir)
files, err := ListDumpFiles(dir, nil)
if err != nil {
t.Fatal(err)
}
if len(files) != 1 || !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
t.Fatalf("nil-cached path should validate: got %+v", files[0].Validation)
}
}
controller/internal/backup/appbackup_bridge.go
+3 -2
View File
@@ -14,6 +14,7 @@ package backup
import (
"context"
"log"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
)
@@ -68,8 +69,8 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
return appbackup.ValidateDump(filePath, dbType)
}
func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
return appbackup.ListDumpFiles(dumpDir)
func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
return appbackup.ListDumpFiles(dumpDir, cached)
}
// --- function forwarders (appdata) ---
controller/internal/backup/backup.go
+43 -2
View File
@@ -233,6 +233,8 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
ValidatedAt: time.Now().Format(time.RFC3339),
TableCount: result.Validation.TableCount,
HasHeader: result.Validation.Valid,
Size: result.Validation.FileSize,
ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339),
}
if !result.Validation.Valid {
cache.Error = result.Validation.Error
@@ -467,6 +469,8 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
ValidatedAt: time.Now().Format(time.RFC3339),
TableCount: result.Validation.TableCount,
HasHeader: result.Validation.Valid,
Size: result.Validation.FileSize,
ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339),
}
if !result.Validation.Valid {
cache.Error = result.Validation.Error
@@ -478,13 +482,50 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
}
// listAllDumpFiles scans per-drive per-stack DB dump directories.
//
// M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is
// passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min
// cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime —
// so the write-back (and its settings.json disk write) only happens when a dump actually changed.
func (m *Manager) listAllDumpFiles() []DumpFileInfo {
modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) }
var cacheSnapshot map[string]settings.DBValidationCache
if m.settings != nil {
cacheSnapshot = m.settings.GetDBValidations()
}
lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
c, ok := cacheSnapshot[name]
if !ok || c.Size != size || c.ModTime != modKey(mod) {
return DumpValidation{}, false // cache miss / changed → validate
}
return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true
}
var allFiles []DumpFileInfo
for drive, stacks := range m.groupStacksByDrive() {
for _, stack := range stacks {
dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name)
if files, err := ListDumpFiles(dumpDir); err == nil {
allFiles = append(allFiles, files...)
files, err := ListDumpFiles(dumpDir, lookup)
if err != nil {
continue
}
for _, f := range files {
// Write back only fresh validations (cache miss against the snapshot), so unchanged
// dumps cause neither a re-validation nor a settings.json write each cycle.
if m.settings != nil {
if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) {
_ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{
ValidatedAt: time.Now().Format(time.RFC3339),
TableCount: f.Validation.TableCount,
HasHeader: f.Validation.Valid,
Error: f.Validation.Error,
Size: f.Size,
ModTime: modKey(f.ModTime),
})
}
}
allFiles = append(allFiles, f)
}
}
}
controller/internal/settings/settings.go
+5
View File
@@ -161,6 +161,11 @@ type DBValidationCache struct {
TableCount int `json:"table_count"`
HasHeader bool `json:"has_header"`
Error string `json:"error,omitempty"`
// M18: Size + ModTime let ListDumpFiles skip the expensive line-by-line re-validation on every
// ~5-min scheduler cycle when the dump file is unchanged. A cache entry is a HIT only when both the
// file size and (RFC3339, second-precision) modtime match the on-disk file.
Size int64 `json:"size,omitempty"`
ModTime string `json:"mod_time,omitempty"` // RFC3339 (UTC)
}
// SetDebug enables or disables debug logging for settings operations.