M18: cache DB-dump validation; skip re-validate on unchanged dumps
ListDumpFiles ran ValidateDump (line-by-line scan) for every dump on every ~5-min RefreshCache cycle — wasted I/O+CPU on large customer dumps. ListDumpFiles now takes an optional cached(name,size,mod) lookup; on a (size+modtime) match it reuses the prior result and skips ValidateDump. settings.DBValidationCache gains Size+ModTime; listAllDumpFiles builds the lookup from the persisted cache and writes back only fresh validations (cache miss), so an unchanged dump triggers neither a re-validation nor a settings.json write each cycle. nil cached = legacy validate-always (back-compat). Tests: cache-hit skips validate (sentinel), cache-miss validates, nil validates.
This commit is contained in:
@@ -14,6 +14,7 @@ package backup
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
)
|
||||
@@ -68,8 +69,8 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
|
||||
return appbackup.ValidateDump(filePath, dbType)
|
||||
}
|
||||
|
||||
func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
|
||||
return appbackup.ListDumpFiles(dumpDir)
|
||||
func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
|
||||
return appbackup.ListDumpFiles(dumpDir, cached)
|
||||
}
|
||||
|
||||
// --- function forwarders (appdata) ---
|
||||
|
||||
@@ -233,6 +233,8 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
|
||||
ValidatedAt: time.Now().Format(time.RFC3339),
|
||||
TableCount: result.Validation.TableCount,
|
||||
HasHeader: result.Validation.Valid,
|
||||
Size: result.Validation.FileSize,
|
||||
ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !result.Validation.Valid {
|
||||
cache.Error = result.Validation.Error
|
||||
@@ -467,6 +469,8 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
|
||||
ValidatedAt: time.Now().Format(time.RFC3339),
|
||||
TableCount: result.Validation.TableCount,
|
||||
HasHeader: result.Validation.Valid,
|
||||
Size: result.Validation.FileSize,
|
||||
ModTime: result.Validation.ModTime.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !result.Validation.Valid {
|
||||
cache.Error = result.Validation.Error
|
||||
@@ -478,13 +482,50 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
|
||||
}
|
||||
|
||||
// listAllDumpFiles scans per-drive per-stack DB dump directories.
|
||||
//
|
||||
// M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is
|
||||
// passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min
|
||||
// cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime —
|
||||
// so the write-back (and its settings.json disk write) only happens when a dump actually changed.
|
||||
func (m *Manager) listAllDumpFiles() []DumpFileInfo {
|
||||
modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) }
|
||||
|
||||
var cacheSnapshot map[string]settings.DBValidationCache
|
||||
if m.settings != nil {
|
||||
cacheSnapshot = m.settings.GetDBValidations()
|
||||
}
|
||||
lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
|
||||
c, ok := cacheSnapshot[name]
|
||||
if !ok || c.Size != size || c.ModTime != modKey(mod) {
|
||||
return DumpValidation{}, false // cache miss / changed → validate
|
||||
}
|
||||
return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true
|
||||
}
|
||||
|
||||
var allFiles []DumpFileInfo
|
||||
for drive, stacks := range m.groupStacksByDrive() {
|
||||
for _, stack := range stacks {
|
||||
dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name)
|
||||
if files, err := ListDumpFiles(dumpDir); err == nil {
|
||||
allFiles = append(allFiles, files...)
|
||||
files, err := ListDumpFiles(dumpDir, lookup)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, f := range files {
|
||||
// Write back only fresh validations (cache miss against the snapshot), so unchanged
|
||||
// dumps cause neither a re-validation nor a settings.json write each cycle.
|
||||
if m.settings != nil {
|
||||
if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) {
|
||||
_ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{
|
||||
ValidatedAt: time.Now().Format(time.RFC3339),
|
||||
TableCount: f.Validation.TableCount,
|
||||
HasHeader: f.Validation.Valid,
|
||||
Error: f.Validation.Error,
|
||||
Size: f.Size,
|
||||
ModTime: modKey(f.ModTime),
|
||||
})
|
||||
}
|
||||
}
|
||||
allFiles = append(allFiles, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user