M18: cache DB-dump validation; skip re-validate on unchanged dumps

ListDumpFiles ran ValidateDump (line-by-line scan) for every dump on every ~5-min
RefreshCache cycle — wasted I/O+CPU on large customer dumps. ListDumpFiles now takes
an optional cached(name,size,mod) lookup; on a (size+modtime) match it reuses the
prior result and skips ValidateDump. settings.DBValidationCache gains Size+ModTime;
listAllDumpFiles builds the lookup from the persisted cache and writes back only fresh
validations (cache miss), so an unchanged dump triggers neither a re-validation nor a
settings.json write each cycle. nil cached = legacy validate-always (back-compat).
Tests: cache-hit skips validate (sentinel), cache-miss validates, nil validates.

controller/internal/appbackup/dbdump.go

@@ -64,7 +64,16 @@ type DumpFileInfo struct {
 }
 
 // DiscoverDatabases finds running database containers via docker ps.
-func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool) ([]DiscoveredDB, error) {
+//
+// knownStacks is the set of actually-deployed stack names; it is used to attribute each DB container to
+// the correct stack (M19). Pass nil/empty for the legacy suffix-strip behaviour.
+func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool, knownStacks []string) ([]DiscoveredDB, error) {
+	known := make(map[string]bool, len(knownStacks))
+	for _, s := range knownStacks {
+		if s != "" {
+			known[s] = true
+		}
+	}
 	if debug {
 		logger.Printf("[DEBUG] DiscoverDatabases: running docker ps to find database containers")
 	}
@@ -113,7 +122,7 @@ func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool) ([]D
 			ContainerID:   id,
 			ContainerName: name,
 			DBType:        dbType,
-			StackName:     deriveStackName(name),
+			StackName:     deriveStackName(name, known),
 		}
 
 		// Get env vars from container
@@ -423,7 +432,12 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
 }
 
 // ListDumpFiles returns info about SQL dump files on disk.
-func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
+//
+// M18: ValidateDump scans the dump line-by-line; on a customer with hundreds-of-MB dumps that is wasted
+// disk I/O + CPU on every ~5-min scheduler cycle. `cached` is an optional lookup that returns a
+// previously-computed DumpValidation for a file whose (name, size, modtime) match — when it returns ok,
+// the expensive ValidateDump is skipped. Pass nil to always validate (legacy fast path / other callers).
+func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
 	entries, err := os.ReadDir(dumpDir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -465,7 +479,14 @@ func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
 			f.StackName = base
 		}
 
-		// Run validation on the file
+		// M18: reuse a cached validation when the file is unchanged (name+size+modtime), else validate.
+		if cached != nil {
+			if v, ok := cached(e.Name(), info.Size(), info.ModTime()); ok {
+				f.Validation = v
+				files = append(files, f)
+				continue
+			}
+		}
 		fullPath := filepath.Join(dumpDir, e.Name())
 		f.Validation = ValidateDump(fullPath, f.DBType)
 
@@ -640,8 +661,51 @@ func getMariaDBPassword(ctx context.Context, containerID string) string {
 	return ""
 }
 
-// deriveStackName strips known DB suffixes from container name.
+// deriveStackName maps a DB container name to its owning stack name.
-func deriveStackName(containerName string) string {
+//
+// M19: the old logic pure-suffix-stripped on `-` (postgres/db/mariadb/mysql/database/redis/cache),
+// which misattributes a stack whose real slug ENDS in a role token (e.g. a stack literally named
+// `my-cache` → stripped to `my`). When the set of actually-deployed stack names is known, cross-reference
+// it so the result is a real stack:
+//   - candidate := suffix-strip result.
+//   - known[candidate]            → candidate (a real DB-role suffix of a real stack, e.g. romm-postgres→romm).
+//   - else known[containerName]   → containerName (the container name IS the stack — don't strip, e.g. my-cache).
+//   - else longest known prefix   → handles <stack>_postgres / <stack>-1 / compose-suffixed names.
+//   - else                        → candidate (fall back to today's suffix-strip; preserves behaviour when
+//                                    the stack list is empty/unavailable, so nothing regresses).
+// A nil/empty `known` map = the legacy fast path (pure suffix-strip).
+func deriveStackName(containerName string, known map[string]bool) string {
+	candidate := suffixStripStackName(containerName)
+
+	if len(known) == 0 {
+		return candidate
+	}
+	if known[candidate] {
+		return candidate
+	}
+	if known[containerName] {
+		return containerName
+	}
+	// Longest known stack name that is a prefix of the container name (tie-break: longest wins).
+	best := ""
+	for name := range known {
+		if name == "" || len(name) >= len(containerName) {
+			continue
+		}
+		// boundary char so "rom" doesn't match "romm-..."; compose/role separators are - or _.
+		sep := containerName[len(name)]
+		if strings.HasPrefix(containerName, name) && (sep == '-' || sep == '_') && len(name) > len(best) {
+			best = name
+		}
+	}
+	if best != "" {
+		return best
+	}
+	return candidate
+}
+
+// suffixStripStackName is the legacy pure suffix-strip (the M19 fallback when no stack list is known).
+func suffixStripStackName(containerName string) string {
 	knownSuffixes := []string{"postgres", "db", "mariadb", "mysql", "database", "redis", "cache"}
 
 	parts := strings.Split(containerName, "-")

controller/internal/appbackup/derivestackname_test.go

@@ -0,0 +1,65 @@
+package appbackup
+
+import "testing"
+
+// TestDeriveStackName_KnownCrossRef asserts M19: deriveStackName cross-references the deployed-stack set
+// so a stack whose slug ends in a DB-role token (e.g. `my-cache`) is NOT misattributed by pure
+// suffix-stripping. The `my-cache` case fails on the pre-fix code (which stripped it to `my`).
+func TestDeriveStackName_KnownCrossRef(t *testing.T) {
+	known := map[string]bool{"romm": true, "my-cache": true, "paperless-ngx": true}
+
+	cases := []struct {
+		container string
+		want      string
+		note      string
+	}{
+		{"romm-postgres", "romm", "role suffix of a real stack → strip"},
+		{"my-cache", "my-cache", "container name IS a real stack → do NOT strip to 'my'"},     // pre-fix: "my"
+		{"my-cache-postgres", "my-cache", "strip role, result is a known stack"},               // pre-fix: "my-cache" (ok) — but via prefix here
+		{"paperless-ngx-postgres", "paperless-ngx", "multi-hyphen stack, role suffix"},
+		{"romm_postgres", "romm", "underscore-separated compose name → longest known prefix"},
+		{"romm-1", "romm", "compose numeric suffix → longest known prefix"},
+	}
+	for _, c := range cases {
+		if got := deriveStackName(c.container, known); got != c.want {
+			t.Errorf("deriveStackName(%q, known) = %q, want %q (%s)", c.container, got, c.want, c.note)
+		}
+	}
+}
+
+// TestDeriveStackName_LegacyFallback asserts the nil/empty-known fast path preserves the old behaviour
+// (pure suffix-strip), so callers without a stack list (e.g. appexport) don't regress.
+func TestDeriveStackName_LegacyFallback(t *testing.T) {
+	cases := []struct {
+		container string
+		want      string
+	}{
+		{"romm-postgres", "romm"},
+		{"paperless-ngx-postgres", "paperless-ngx"},
+		{"my-cache", "my"},        // legacy strips the role-token suffix
+		{"unknown-db", "unknown"}, // legacy strip
+		{"standalone", "standalone"},
+	}
+	for _, c := range cases {
+		if got := deriveStackName(c.container, nil); got != c.want {
+			t.Errorf("deriveStackName(%q, nil) = %q, want %q (legacy)", c.container, got, c.want)
+		}
+		// empty (non-nil) map must behave identically to nil
+		if got := deriveStackName(c.container, map[string]bool{}); got != c.want {
+			t.Errorf("deriveStackName(%q, {}) = %q, want %q (legacy)", c.container, got, c.want)
+		}
+	}
+}
+
+// TestDeriveStackName_UnknownContainerFallsBack asserts that when the container matches no known stack at
+// all, the result falls back to the suffix-strip candidate (no spurious prefix match).
+func TestDeriveStackName_UnknownContainerFallsBack(t *testing.T) {
+	known := map[string]bool{"romm": true}
+	if got := deriveStackName("grafana-db", known); got != "grafana" {
+		t.Fatalf("deriveStackName(grafana-db, {romm}) = %q, want grafana (fallback strip)", got)
+	}
+	// must NOT match "romm" as a prefix of an unrelated name
+	if got := deriveStackName("rommother-db", known); got != "rommother" {
+		t.Fatalf("deriveStackName(rommother-db) = %q, want rommother (no false prefix match)", got)
+	}
+}

controller/internal/appbackup/listdumpfiles_cache_test.go

@@ -0,0 +1,94 @@
+package appbackup
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// writeDumpFile creates a minimal VALID postgres dump so a real ValidateDump would set Valid=true and
+// TableCount=1 — distinguishable from the sentinel the cache returns.
+func writeDumpFile(t *testing.T, dir string) string {
+	t.Helper()
+	path := filepath.Join(dir, "romm-postgres.sql")
+	body := "-- PostgreSQL database dump\n" +
+		"CREATE TABLE public.t (id int);\n" +
+		"-- PostgreSQL database dump complete\n" +
+		// pad past the 100-byte floor
+		"-- padding ------------------------------------------------------------\n"
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+// TestListDumpFiles_CacheHitSkipsValidate asserts M18: when the `cached` lookup returns ok for an
+// unchanged file, ListDumpFiles reuses that result and does NOT re-run the (expensive) ValidateDump.
+// Proven via a sentinel TableCount=999 that only the cache could supply (a real validate yields 1).
+func TestListDumpFiles_CacheHitSkipsValidate(t *testing.T) {
+	dir := t.TempDir()
+	writeDumpFile(t, dir)
+
+	const sentinel = 999
+	calls := 0
+	cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
+		calls++
+		return DumpValidation{Valid: true, TableCount: sentinel}, true // always "hit"
+	}
+
+	files, err := ListDumpFiles(dir, cached)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("expected 1 dump file, got %d", len(files))
+	}
+	if files[0].Validation.TableCount != sentinel {
+		t.Fatalf("validation TableCount = %d, want %d (sentinel from cache) — ValidateDump was re-run instead of using the cache",
+			files[0].Validation.TableCount, sentinel)
+	}
+	if calls != 1 {
+		t.Fatalf("cached lookup called %d times, want 1", calls)
+	}
+}
+
+// TestListDumpFiles_CacheMissValidates asserts that on a cache MISS (e.g. modtime changed) ListDumpFiles
+// falls through to a real ValidateDump (sentinel must NOT appear; real TableCount=1).
+func TestListDumpFiles_CacheMissValidates(t *testing.T) {
+	dir := t.TempDir()
+	writeDumpFile(t, dir)
+
+	cached := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
+		return DumpValidation{Valid: true, TableCount: 999}, false // always "miss"
+	}
+
+	files, err := ListDumpFiles(dir, cached)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("expected 1 dump file, got %d", len(files))
+	}
+	if files[0].Validation.TableCount == 999 {
+		t.Fatalf("got the sentinel on a cache MISS — should have run a real ValidateDump")
+	}
+	if !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
+		t.Fatalf("real validation expected Valid=true TableCount=1, got Valid=%v TableCount=%d",
+			files[0].Validation.Valid, files[0].Validation.TableCount)
+	}
+}
+
+// TestListDumpFiles_NilCachedAlwaysValidates asserts the legacy fast path (cached==nil) still validates.
+func TestListDumpFiles_NilCachedAlwaysValidates(t *testing.T) {
+	dir := t.TempDir()
+	writeDumpFile(t, dir)
+
+	files, err := ListDumpFiles(dir, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 1 || !files[0].Validation.Valid || files[0].Validation.TableCount != 1 {
+		t.Fatalf("nil-cached path should validate: got %+v", files[0].Validation)
+	}
+}

controller/internal/appexport/export.go

@@ -510,7 +510,7 @@ func (e *Exporter) dumpDatabase(stackName, dbDir string, manifest *Manifest) boo
 	defer cancel()
 
 	e.debugf("discovering databases (looking for stack %s)...", stackName)
-	dbs, err := appbackup.DiscoverDatabases(ctx, e.logger, e.debug)
+	dbs, err := appbackup.DiscoverDatabases(ctx, e.logger, e.debug, nil)
 	if err != nil {
 		e.logger.Printf("[WARN] Export: DB discovery error: %v", err)
 		return false

controller/internal/backup/appbackup_bridge.go

@@ -14,6 +14,7 @@ package backup
 import (
 	"context"
 	"log"
+	"time"
 
 	"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
 )
@@ -47,8 +48,8 @@ const FelhomDataDir = appbackup.FelhomDataDir
 
 // --- function forwarders (dbdump) ---
 
-func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool) ([]DiscoveredDB, error) {
+func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool, knownStacks []string) ([]DiscoveredDB, error) {
-	return appbackup.DiscoverDatabases(ctx, logger, debug)
+	return appbackup.DiscoverDatabases(ctx, logger, debug, knownStacks)
 }
 
 func DumpAll(ctx context.Context, dbs []DiscoveredDB, dumpDir string, logger *log.Logger, debug bool) []DumpResult {
@@ -68,8 +69,8 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
 	return appbackup.ValidateDump(filePath, dbType)
 }
 
-func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) {
+func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
-	return appbackup.ListDumpFiles(dumpDir)
+	return appbackup.ListDumpFiles(dumpDir, cached)
 }
 
 // --- function forwarders (appdata) ---

controller/internal/backup/backup.go

@@ -122,6 +122,20 @@ func (m *Manager) AppNamespaceRoot(stackName string) string {
 	return m.namespaceRoot(drivePath)
 }
 
+// knownStackNames returns the names of all deployed stacks, for M19 DB-container→stack attribution.
+// Empty when no provider is wired (DiscoverDatabases then falls back to legacy suffix-strip).
+func (m *Manager) knownStackNames() []string {
+	if m.stackProvider == nil {
+		return nil
+	}
+	stacks := m.stackProvider.ListDeployedStacks()
+	names := make([]string, 0, len(stacks))
+	for _, s := range stacks {
+		names = append(names, s.Name)
+	}
+	return names
+}
+
 // groupStacksByDrive groups deployed stacks by their home drive path.
 func (m *Manager) groupStacksByDrive() map[string][]StackSummary {
 	if m.stackProvider == nil {
@@ -158,7 +172,7 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
 	start := time.Now()
 	m.logger.Printf("[INFO] [backup] Starting database dump run")
 
-	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug())
+	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames())
 	if err != nil {
 		m.logger.Printf("[ERROR] [backup] Database discovery failed: %v", err)
 		return err
@@ -219,6 +233,8 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
 					ValidatedAt: time.Now().Format(time.RFC3339),
 					TableCount:  result.Validation.TableCount,
 					HasHeader:   result.Validation.Valid,
+					Size:        result.Validation.FileSize,
+					ModTime:     result.Validation.ModTime.UTC().Format(time.RFC3339),
 				}
 				if !result.Validation.Valid {
 					cache.Error = result.Validation.Error
@@ -415,7 +431,7 @@ func (m *Manager) GetStackHDDMounts(name string) []string {
 // DumpStackDB runs a database dump for containers belonging to a specific stack.
 // Dumps to the stack's home drive: <drive>/backups/primary/<stack>/db-dumps/.
 func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
-	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug())
+	dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames())
 	if err != nil {
 		return fmt.Errorf("database discovery failed: %w", err)
 	}
@@ -453,6 +469,8 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
 				ValidatedAt: time.Now().Format(time.RFC3339),
 				TableCount:  result.Validation.TableCount,
 				HasHeader:   result.Validation.Valid,
+				Size:        result.Validation.FileSize,
+				ModTime:     result.Validation.ModTime.UTC().Format(time.RFC3339),
 			}
 			if !result.Validation.Valid {
 				cache.Error = result.Validation.Error
@@ -464,13 +482,50 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
 }
 
 // listAllDumpFiles scans per-drive per-stack DB dump directories.
+//
+// M18: a snapshot of the persisted validation cache (keyed by filename, matched on size+modtime) is
+// passed to ListDumpFiles so an UNCHANGED dump is not re-validated (line-by-line scan) on every ~5-min
+// cycle. Freshly-validated dumps (cache miss) are written back to settings, keyed by name+size+modtime —
+// so the write-back (and its settings.json disk write) only happens when a dump actually changed.
 func (m *Manager) listAllDumpFiles() []DumpFileInfo {
+	modKey := func(t time.Time) string { return t.UTC().Format(time.RFC3339) }
+
+	var cacheSnapshot map[string]settings.DBValidationCache
+	if m.settings != nil {
+		cacheSnapshot = m.settings.GetDBValidations()
+	}
+	lookup := func(name string, size int64, mod time.Time) (DumpValidation, bool) {
+		c, ok := cacheSnapshot[name]
+		if !ok || c.Size != size || c.ModTime != modKey(mod) {
+			return DumpValidation{}, false // cache miss / changed → validate
+		}
+		return DumpValidation{Valid: c.HasHeader, TableCount: c.TableCount, Error: c.Error, FileSize: size, ModTime: mod}, true
+	}
+
 	var allFiles []DumpFileInfo
 	for drive, stacks := range m.groupStacksByDrive() {
 		for _, stack := range stacks {
 			dumpDir := AppDBDumpPath(m.namespaceRoot(drive), stack.Name)
-			if files, err := ListDumpFiles(dumpDir); err == nil {
+			files, err := ListDumpFiles(dumpDir, lookup)
-				allFiles = append(allFiles, files...)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				// Write back only fresh validations (cache miss against the snapshot), so unchanged
+				// dumps cause neither a re-validation nor a settings.json write each cycle.
+				if m.settings != nil {
+					if c, ok := cacheSnapshot[f.FileName]; !ok || c.Size != f.Size || c.ModTime != modKey(f.ModTime) {
+						_ = m.settings.SetDBValidation(f.FileName, settings.DBValidationCache{
+							ValidatedAt: time.Now().Format(time.RFC3339),
+							TableCount:  f.Validation.TableCount,
+							HasHeader:   f.Validation.Valid,
+							Error:       f.Validation.Error,
+							Size:        f.Size,
+							ModTime:     modKey(f.ModTime),
+						})
+					}
+				}
+				allFiles = append(allFiles, f)
 			}
 		}
 	}
@@ -492,7 +547,7 @@ func (m *Manager) RefreshCache(nextDBDump time.Time) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	if dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug()); err == nil {
+	if dbs, err := DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()); err == nil {
 		status.DiscoveredDBs = dbs
 	}
 

controller/internal/backup/restore_db.go

@@ -41,7 +41,7 @@ func (m *Manager) reimportDBDumps(ctx context.Context, stackName, nsRoot string)
 	discover := m.discoverDBs
 	if discover == nil {
 		discover = func(ctx context.Context) ([]DiscoveredDB, error) {
-			return DiscoverDatabases(ctx, m.logger, m.isDebug())
+			return DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames())
 		}
 	}
 	imp := m.importDBDump

controller/internal/settings/settings.go

@@ -161,6 +161,11 @@ type DBValidationCache struct {
 	TableCount  int    `json:"table_count"`
 	HasHeader   bool   `json:"has_header"`
 	Error       string `json:"error,omitempty"`
+	// M18: Size + ModTime let ListDumpFiles skip the expensive line-by-line re-validation on every
+	// ~5-min scheduler cycle when the dump file is unchanged. A cache entry is a HIT only when both the
+	// file size and (RFC3339, second-precision) modtime match the on-disk file.
+	Size    int64  `json:"size,omitempty"`
+	ModTime string `json:"mod_time,omitempty"` // RFC3339 (UTC)
 }
 
 // SetDebug enables or disables debug logging for settings operations.