admin
f8afe5c055
ListDumpFiles ran ValidateDump (line-by-line scan) for every dump on every ~5-min RefreshCache cycle — wasted I/O+CPU on large customer dumps. ListDumpFiles now takes an optional cached(name,size,mod) lookup; on a (size+modtime) match it reuses the prior result and skips ValidateDump. settings.DBValidationCache gains Size+ModTime; listAllDumpFiles builds the lookup from the persisted cache and writes back only fresh validations (cache miss), so an unchanged dump triggers neither a re-validation nor a settings.json write each cycle. nil cached = legacy validate-always (back-compat). Tests: cache-hit skips validate (sentinel), cache-miss validates, nil validates.
749 lines
22 KiB
Go
749 lines
22 KiB
Go
package appbackup
|
|
|
|
import (
|
|
"bufio"
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
)
|
|
|
|
// DBType represents a database engine type.
|
|
type DBType string
|
|
|
|
const (
|
|
DBTypePostgres DBType = "postgres"
|
|
DBTypeMariaDB DBType = "mariadb"
|
|
)
|
|
|
|
// DiscoveredDB holds metadata about a running database container.
|
|
type DiscoveredDB struct {
|
|
ContainerName string
|
|
ContainerID string
|
|
DBType DBType
|
|
DBUser string
|
|
DBName string
|
|
StackName string
|
|
}
|
|
|
|
// DumpResult holds the outcome of a single database dump.
|
|
type DumpResult struct {
|
|
DB DiscoveredDB
|
|
FilePath string
|
|
Size int64
|
|
Duration time.Duration
|
|
Error error
|
|
Validation DumpValidation
|
|
}
|
|
|
|
// DumpValidation holds the result of a dump file structural check.
|
|
type DumpValidation struct {
|
|
Valid bool
|
|
TableCount int
|
|
Error string
|
|
FileSize int64
|
|
ModTime time.Time
|
|
}
|
|
|
|
// DumpFileInfo holds info about a dump file on disk.
|
|
type DumpFileInfo struct {
|
|
FileName string
|
|
StackName string
|
|
DBType DBType
|
|
Size int64
|
|
ModTime time.Time
|
|
Validation DumpValidation
|
|
}
|
|
|
|
// DiscoverDatabases finds running database containers via docker ps.
|
|
//
|
|
// knownStacks is the set of actually-deployed stack names; it is used to attribute each DB container to
|
|
// the correct stack (M19). Pass nil/empty for the legacy suffix-strip behaviour.
|
|
func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool, knownStacks []string) ([]DiscoveredDB, error) {
|
|
known := make(map[string]bool, len(knownStacks))
|
|
for _, s := range knownStacks {
|
|
if s != "" {
|
|
known[s] = true
|
|
}
|
|
}
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: running docker ps to find database containers")
|
|
}
|
|
cmd := exec.CommandContext(ctx, "docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}", "--filter", "status=running")
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("docker ps failed: %w", err)
|
|
}
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: docker ps output: %s", util.TruncateStr(strings.TrimSpace(string(out)), 500))
|
|
}
|
|
|
|
var dbs []DiscoveredDB
|
|
var skipped int
|
|
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
parts := strings.SplitN(line, "\t", 3)
|
|
if len(parts) < 3 {
|
|
continue
|
|
}
|
|
|
|
id, name, image := parts[0], parts[1], strings.ToLower(parts[2])
|
|
|
|
var dbType DBType
|
|
if strings.Contains(image, "postgres") {
|
|
dbType = DBTypePostgres
|
|
} else if strings.Contains(image, "mariadb") || strings.Contains(image, "mysql") {
|
|
dbType = DBTypeMariaDB
|
|
} else {
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: skipping container %s (image=%s, not a database)", name, image)
|
|
}
|
|
skipped++
|
|
continue
|
|
}
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: found %s container: %s (id=%s)", dbType, name, id[:12])
|
|
}
|
|
|
|
db := DiscoveredDB{
|
|
ContainerID: id,
|
|
ContainerName: name,
|
|
DBType: dbType,
|
|
StackName: deriveStackName(name, known),
|
|
}
|
|
|
|
// Get env vars from container
|
|
if err := populateDBEnv(ctx, &db); err != nil {
|
|
logger.Printf("[WARN] [backup] Could not read env vars for %s: %v", name, err)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: skipping %s — env read failed", name)
|
|
}
|
|
continue
|
|
}
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: %s → stack=%s, dbUser=%s, dbName=%s", name, db.StackName, db.DBUser, db.DBName)
|
|
}
|
|
|
|
dbs = append(dbs, db)
|
|
}
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DiscoverDatabases: found %d database(s), skipped %d non-DB container(s)", len(dbs), skipped)
|
|
}
|
|
|
|
logger.Printf("[INFO] [backup] Discovered %d databases", len(dbs))
|
|
return dbs, nil
|
|
}
|
|
|
|
// DumpAll dumps all discovered databases.
|
|
func DumpAll(ctx context.Context, dbs []DiscoveredDB, dumpDir string, logger *log.Logger, debug bool) []DumpResult {
|
|
// Clean up old .tmp files (older than 1 hour)
|
|
cleanupTmpFiles(dumpDir, logger)
|
|
|
|
logger.Printf("[INFO] [backup] Starting DB dump for %d databases", len(dbs))
|
|
var results []DumpResult
|
|
var failed int
|
|
for _, db := range dbs {
|
|
result := DumpOne(ctx, db, dumpDir, logger, debug)
|
|
results = append(results, result)
|
|
if result.Error != nil {
|
|
failed++
|
|
}
|
|
}
|
|
logger.Printf("[INFO] [backup] DB dump complete: %d succeeded, %d failed", len(results)-failed, failed)
|
|
return results
|
|
}
|
|
|
|
// DumpOne dumps a single database.
|
|
func DumpOne(ctx context.Context, db DiscoveredDB, dumpDir string, logger *log.Logger, debug bool) DumpResult {
|
|
start := time.Now()
|
|
result := DumpResult{DB: db}
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: starting dump for container=%s, stack=%s, dbType=%s, dumpDir=%s",
|
|
db.ContainerName, db.StackName, db.DBType, dumpDir)
|
|
}
|
|
|
|
// Ensure dump directory exists
|
|
if err := os.MkdirAll(dumpDir, 0755); err != nil {
|
|
result.Error = fmt.Errorf("creating dump dir: %w", err)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
|
|
filename := fmt.Sprintf("%s-%s.sql", db.StackName, db.DBType)
|
|
tmpPath := filepath.Join(dumpDir, filename+".tmp")
|
|
finalPath := filepath.Join(dumpDir, filename)
|
|
|
|
// 5-minute timeout per dump
|
|
dumpCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
|
defer cancel()
|
|
|
|
// Verify container is still running
|
|
checkCmd := exec.CommandContext(dumpCtx, "docker", "inspect", "--format", "{{.State.Running}}", db.ContainerID)
|
|
checkOut, err := checkCmd.Output()
|
|
if err != nil || strings.TrimSpace(string(checkOut)) != "true" {
|
|
result.Error = fmt.Errorf("container %s no longer running", db.ContainerName)
|
|
result.Duration = time.Since(start)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: container %s is no longer running — skipping", db.ContainerName)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// Build dump command
|
|
var cmd *exec.Cmd
|
|
switch db.DBType {
|
|
case DBTypePostgres:
|
|
cmd = exec.CommandContext(dumpCtx, "docker", "exec", db.ContainerID,
|
|
"pg_dump", "-U", db.DBUser, "-d", db.DBName,
|
|
"--clean", "--if-exists", "--no-owner", "--no-privileges")
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: pg_dump command: docker exec %s pg_dump -U %s -d %s --clean --if-exists --no-owner --no-privileges",
|
|
db.ContainerID[:12], db.DBUser, db.DBName)
|
|
}
|
|
case DBTypeMariaDB:
|
|
// Get root password from container env
|
|
password := getMariaDBPassword(dumpCtx, db.ContainerID)
|
|
if password == "" {
|
|
result.Error = fmt.Errorf("could not determine MariaDB root password for %s", db.ContainerName)
|
|
result.Duration = time.Since(start)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: MariaDB root password not found for %s — skipping", db.ContainerName)
|
|
}
|
|
return result
|
|
}
|
|
cmd = exec.CommandContext(dumpCtx, "docker", "exec", db.ContainerID,
|
|
"mariadb-dump", "-u", "root", "-p***",
|
|
"--single-transaction", "--routines", "--triggers", db.DBName)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: mariadb-dump command: docker exec %s mariadb-dump -u root -p*** --single-transaction --routines --triggers %s",
|
|
db.ContainerID[:12], db.DBName)
|
|
}
|
|
// Actual command with real password (not logged)
|
|
cmd = exec.CommandContext(dumpCtx, "docker", "exec", db.ContainerID,
|
|
"mariadb-dump", "-u", "root", "-p"+password,
|
|
"--single-transaction", "--routines", "--triggers", db.DBName)
|
|
default:
|
|
result.Error = fmt.Errorf("unsupported DB type: %s", db.DBType)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
|
|
// Write output to tmp file
|
|
tmpFile, err := os.Create(tmpPath)
|
|
if err != nil {
|
|
result.Error = fmt.Errorf("creating tmp file: %w", err)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
defer tmpFile.Close()
|
|
|
|
cmd.Stdout = tmpFile
|
|
var stderr strings.Builder
|
|
cmd.Stderr = &stderr
|
|
|
|
err = cmd.Run()
|
|
|
|
if err != nil {
|
|
os.Remove(tmpPath)
|
|
errMsg := stderr.String()
|
|
if len(errMsg) > 200 {
|
|
errMsg = errMsg[:200]
|
|
}
|
|
result.Error = fmt.Errorf("dump failed: %v — %s", err, errMsg)
|
|
result.Duration = time.Since(start)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: dump command failed for %s: %v", db.ContainerName, result.Error)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// Close and sync tmpFile before rename to ensure data is flushed to disk (H8 fix).
|
|
if err := tmpFile.Sync(); err != nil {
|
|
os.Remove(tmpPath)
|
|
result.Error = fmt.Errorf("syncing dump file: %w", err)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
if err := tmpFile.Close(); err != nil {
|
|
os.Remove(tmpPath)
|
|
result.Error = fmt.Errorf("closing dump file: %w", err)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
|
|
// Check file size
|
|
stat, err := os.Stat(tmpPath)
|
|
if err != nil || stat.Size() == 0 {
|
|
os.Remove(tmpPath)
|
|
result.Error = fmt.Errorf("dump produced empty file for %s", db.ContainerName)
|
|
result.Duration = time.Since(start)
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: dump produced empty file for %s", db.ContainerName)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// Rename tmp to final
|
|
if err := os.Rename(tmpPath, finalPath); err != nil {
|
|
os.Remove(tmpPath)
|
|
result.Error = fmt.Errorf("renaming dump file: %w", err)
|
|
result.Duration = time.Since(start)
|
|
return result
|
|
}
|
|
|
|
result.FilePath = finalPath
|
|
result.Size = stat.Size()
|
|
result.Duration = time.Since(start)
|
|
|
|
// Run validation on the dump file
|
|
result.Validation = ValidateDump(finalPath, db.DBType)
|
|
|
|
if debug {
|
|
logger.Printf("[DEBUG] DumpOne: completed %s → %s (size=%s, valid=%v, tables=%d, duration=%s)",
|
|
db.ContainerName, filename, humanizeBytes(stat.Size()),
|
|
result.Validation.Valid, result.Validation.TableCount, result.Duration.Round(time.Millisecond))
|
|
}
|
|
|
|
logger.Printf("[INFO] [backup] DB dump: %s → %s (%s, %s, %d tables)", db.ContainerName, filename,
|
|
humanizeBytes(stat.Size()), result.Duration.Round(time.Millisecond), result.Validation.TableCount)
|
|
|
|
return result
|
|
}
|
|
|
|
// ValidateDump checks a SQL dump file for basic structural integrity.
|
|
func ValidateDump(filePath string, dbType DBType) DumpValidation {
|
|
stat, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return DumpValidation{Error: fmt.Sprintf("stat failed: %v", err)}
|
|
}
|
|
|
|
v := DumpValidation{
|
|
FileSize: stat.Size(),
|
|
ModTime: stat.ModTime(),
|
|
}
|
|
|
|
if stat.Size() < 100 {
|
|
v.Error = "dump file too small (< 100 bytes)"
|
|
log.Printf("[WARN] [backup] ValidateDump FAIL: %s — %s", filePath, v.Error)
|
|
return v
|
|
}
|
|
|
|
// H1: Use bufio.Scanner to read line-by-line instead of loading entire file into memory.
|
|
// Large dumps (500MB+) would cause massive allocations on every 5-min cache refresh.
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
v.Error = fmt.Sprintf("read failed: %v", err)
|
|
log.Printf("[WARN] [backup] ValidateDump FAIL: %s — %s", filePath, v.Error)
|
|
return v
|
|
}
|
|
defer f.Close()
|
|
|
|
// Use bufio.Reader instead of Scanner: ReadLine gracefully handles lines
|
|
// longer than the buffer (isPrefix=true) so we can skip them. Only short
|
|
// lines matter (headers, CREATE TABLE). Long COPY/INSERT data lines
|
|
// (e.g., Immich's binary-encoded image data) are skipped without allocating.
|
|
reader := bufio.NewReaderSize(f, 256*1024)
|
|
|
|
lineNum := 0
|
|
headerFound := false
|
|
tableCount := 0
|
|
for {
|
|
lineBytes, isPrefix, err := reader.ReadLine()
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
v.Error = fmt.Sprintf("hiba az olvasás közben: %v", err)
|
|
log.Printf("[WARN] [backup] ValidateDump FAIL: %s — read error: %v", filePath, err)
|
|
return v
|
|
}
|
|
break // EOF
|
|
}
|
|
if isPrefix {
|
|
// Line exceeds buffer — skip remainder (COPY data, large INSERTs)
|
|
for isPrefix && err == nil {
|
|
_, isPrefix, err = reader.ReadLine()
|
|
}
|
|
continue
|
|
}
|
|
|
|
line := string(lineBytes)
|
|
lineNum++
|
|
|
|
// Header check — scan first 10 lines for expected dump header
|
|
// MariaDB 11.4+ prepends a sandbox comment before the header line
|
|
if lineNum <= 10 && !headerFound {
|
|
switch dbType {
|
|
case DBTypeMariaDB:
|
|
if strings.HasPrefix(line, "-- MariaDB dump") ||
|
|
strings.HasPrefix(line, "-- MySQL dump") ||
|
|
strings.HasPrefix(line, "-- mysqldump") {
|
|
headerFound = true
|
|
}
|
|
case DBTypePostgres:
|
|
if strings.HasPrefix(line, "-- PostgreSQL database dump") {
|
|
headerFound = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Count CREATE TABLE statements
|
|
upper := strings.ToUpper(strings.TrimSpace(line))
|
|
if strings.HasPrefix(upper, "CREATE TABLE") {
|
|
tableCount++
|
|
}
|
|
}
|
|
|
|
v.TableCount = tableCount
|
|
|
|
if !headerFound {
|
|
switch dbType {
|
|
case DBTypeMariaDB:
|
|
v.Error = "MariaDB dump missing comment header"
|
|
case DBTypePostgres:
|
|
v.Error = "PostgreSQL dump missing comment header"
|
|
}
|
|
log.Printf("[WARN] [backup] ValidateDump FAIL: %s — %s", filePath, v.Error)
|
|
return v
|
|
}
|
|
|
|
if tableCount == 0 {
|
|
v.Error = "no CREATE TABLE statements found"
|
|
log.Printf("[WARN] [backup] ValidateDump FAIL: %s — %s (header was found, scanned %d lines)", filePath, v.Error, lineNum)
|
|
return v
|
|
}
|
|
|
|
v.Valid = true
|
|
return v
|
|
}
|
|
|
|
// ListDumpFiles returns info about SQL dump files on disk.
|
|
//
|
|
// M18: ValidateDump scans the dump line-by-line; on a customer with hundreds-of-MB dumps that is wasted
|
|
// disk I/O + CPU on every ~5-min scheduler cycle. `cached` is an optional lookup that returns a
|
|
// previously-computed DumpValidation for a file whose (name, size, modtime) match — when it returns ok,
|
|
// the expensive ValidateDump is skipped. Pass nil to always validate (legacy fast path / other callers).
|
|
func ListDumpFiles(dumpDir string, cached func(name string, size int64, mod time.Time) (DumpValidation, bool)) ([]DumpFileInfo, error) {
|
|
entries, err := os.ReadDir(dumpDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, fmt.Errorf("reading dump dir: %w", err)
|
|
}
|
|
|
|
var files []DumpFileInfo
|
|
for _, e := range entries {
|
|
// M2: Check .tmp before .sql to correctly skip ".sql.tmp" temp files (was dead code before).
|
|
if e.IsDir() || strings.HasSuffix(e.Name(), ".tmp") {
|
|
continue
|
|
}
|
|
if !strings.HasSuffix(e.Name(), ".sql") {
|
|
continue
|
|
}
|
|
|
|
info, err := e.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
f := DumpFileInfo{
|
|
FileName: e.Name(),
|
|
Size: info.Size(),
|
|
ModTime: info.ModTime(),
|
|
}
|
|
|
|
// Parse stack name and DB type from filename: "paperless-ngx-postgres.sql"
|
|
base := strings.TrimSuffix(e.Name(), ".sql")
|
|
if strings.HasSuffix(base, "-postgres") {
|
|
f.StackName = strings.TrimSuffix(base, "-postgres")
|
|
f.DBType = DBTypePostgres
|
|
} else if strings.HasSuffix(base, "-mariadb") {
|
|
f.StackName = strings.TrimSuffix(base, "-mariadb")
|
|
f.DBType = DBTypeMariaDB
|
|
} else {
|
|
f.StackName = base
|
|
}
|
|
|
|
// M18: reuse a cached validation when the file is unchanged (name+size+modtime), else validate.
|
|
if cached != nil {
|
|
if v, ok := cached(e.Name(), info.Size(), info.ModTime()); ok {
|
|
f.Validation = v
|
|
files = append(files, f)
|
|
continue
|
|
}
|
|
}
|
|
fullPath := filepath.Join(dumpDir, e.Name())
|
|
f.Validation = ValidateDump(fullPath, f.DBType)
|
|
|
|
files = append(files, f)
|
|
}
|
|
|
|
return files, nil
|
|
}
|
|
|
|
func populateDBEnv(ctx context.Context, db *DiscoveredDB) error {
|
|
cmd := exec.CommandContext(ctx, "docker", "inspect", db.ContainerID,
|
|
"--format", "{{range .Config.Env}}{{println .}}{{end}}")
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
env := make(map[string]string)
|
|
for _, line := range strings.Split(string(out), "\n") {
|
|
if idx := strings.IndexByte(line, '='); idx > 0 {
|
|
env[line[:idx]] = line[idx+1:]
|
|
}
|
|
}
|
|
|
|
switch db.DBType {
|
|
case DBTypePostgres:
|
|
db.DBUser = env["POSTGRES_USER"]
|
|
if db.DBUser == "" {
|
|
db.DBUser = "postgres"
|
|
}
|
|
db.DBName = env["POSTGRES_DB"]
|
|
if db.DBName == "" {
|
|
db.DBName = db.DBUser
|
|
}
|
|
case DBTypeMariaDB:
|
|
db.DBName = env["MYSQL_DATABASE"]
|
|
if db.DBName == "" {
|
|
db.DBName = env["MARIADB_DATABASE"]
|
|
}
|
|
if db.DBName == "" {
|
|
db.DBName = "mysql" // fallback to dump all
|
|
}
|
|
db.DBUser = "root"
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ImportDump replays a (possibly gzipped) SQL dump into a RUNNING database container — the read-side
|
|
// counterpart to DumpOne (F17). It reuses the per-engine clients (psql / mariadb) and the DiscoveredDB's
|
|
// OWN credentials (discovered from the live container env), so the caller needs no external env map. The
|
|
// container must already be running (the restore flow brings the stack up first); ImportDump briefly
|
|
// waits for the engine to accept connections, then pipes the dump in. The backup dumps are produced with
|
|
// DROP/CREATE (pg_dump --clean --if-exists; mariadb-dump's default --add-drop-table), so a replay fully
|
|
// reconstructs the captured logical state.
|
|
func ImportDump(ctx context.Context, db DiscoveredDB, dumpPath string, logger *log.Logger, debug bool) error {
|
|
if err := waitDBReady(ctx, db, 30*time.Second); err != nil {
|
|
return fmt.Errorf("waiting for %s (%s) readiness: %w", db.ContainerName, db.DBType, err)
|
|
}
|
|
|
|
f, err := os.Open(dumpPath)
|
|
if err != nil {
|
|
return fmt.Errorf("opening dump %s: %w", dumpPath, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
var reader io.Reader = f
|
|
if strings.HasSuffix(dumpPath, ".gz") {
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return fmt.Errorf("opening gzip %s: %w", dumpPath, err)
|
|
}
|
|
defer gr.Close()
|
|
reader = gr
|
|
}
|
|
|
|
impCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
defer cancel()
|
|
|
|
var cmd *exec.Cmd
|
|
switch db.DBType {
|
|
case DBTypePostgres:
|
|
user := db.DBUser
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
dbName := db.DBName
|
|
if dbName == "" {
|
|
dbName = user
|
|
}
|
|
// ON_ERROR_STOP=1: a real import error must FAIL (and surface), not silently half-apply.
|
|
cmd = exec.CommandContext(impCtx, "docker", "exec", "-i", db.ContainerID,
|
|
"psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", dbName)
|
|
case DBTypeMariaDB:
|
|
password := getMariaDBPassword(impCtx, db.ContainerID)
|
|
if password == "" {
|
|
return fmt.Errorf("could not determine MariaDB root password for %s", db.ContainerName)
|
|
}
|
|
cmd = exec.CommandContext(impCtx, "docker", "exec", "-i", db.ContainerID,
|
|
"mariadb", "-u", "root", "-p"+password, db.DBName)
|
|
default:
|
|
return fmt.Errorf("unsupported DB type: %s", db.DBType)
|
|
}
|
|
|
|
cmd.Stdin = reader
|
|
var stderr strings.Builder
|
|
cmd.Stderr = &stderr
|
|
if debug && logger != nil {
|
|
logger.Printf("[DEBUG] [backup] ImportDump: importing %s into %s (%s)", dumpPath, db.ContainerName, db.DBType)
|
|
}
|
|
if err := cmd.Run(); err != nil {
|
|
msg := strings.TrimSpace(stderr.String())
|
|
if len(msg) > 300 {
|
|
msg = msg[:300]
|
|
}
|
|
return fmt.Errorf("%s import into %s failed: %s — %w", db.DBType, db.ContainerName, msg, err)
|
|
}
|
|
if logger != nil {
|
|
logger.Printf("[INFO] [backup] Imported DB dump %s into %s (%s)", filepath.Base(dumpPath), db.ContainerName, db.DBType)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// waitDBReady polls until the database accepts connections (pg_isready / mariadb-admin ping).
|
|
func waitDBReady(ctx context.Context, db DiscoveredDB, timeout time.Duration) error {
|
|
deadline := time.Now().Add(timeout)
|
|
for {
|
|
c, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
var cmd *exec.Cmd
|
|
switch db.DBType {
|
|
case DBTypePostgres:
|
|
user := db.DBUser
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
cmd = exec.CommandContext(c, "docker", "exec", db.ContainerID, "pg_isready", "-U", user)
|
|
case DBTypeMariaDB:
|
|
pw := getMariaDBPassword(c, db.ContainerID)
|
|
cmd = exec.CommandContext(c, "docker", "exec", db.ContainerID, "mariadb-admin", "ping", "-u", "root", "-p"+pw)
|
|
default:
|
|
cancel()
|
|
return fmt.Errorf("unsupported DB type: %s", db.DBType)
|
|
}
|
|
err := cmd.Run()
|
|
cancel()
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("timeout after %s", timeout)
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
}
|
|
|
|
func getMariaDBPassword(ctx context.Context, containerID string) string {
|
|
cmd := exec.CommandContext(ctx, "docker", "inspect", containerID,
|
|
"--format", "{{range .Config.Env}}{{println .}}{{end}}")
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
|
|
for _, line := range strings.Split(string(out), "\n") {
|
|
if strings.HasPrefix(line, "MYSQL_ROOT_PASSWORD=") {
|
|
return strings.TrimPrefix(line, "MYSQL_ROOT_PASSWORD=")
|
|
}
|
|
if strings.HasPrefix(line, "MARIADB_ROOT_PASSWORD=") {
|
|
return strings.TrimPrefix(line, "MARIADB_ROOT_PASSWORD=")
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// deriveStackName maps a DB container name to its owning stack name.
|
|
//
|
|
// M19: the old logic pure-suffix-stripped on `-` (postgres/db/mariadb/mysql/database/redis/cache),
|
|
// which misattributes a stack whose real slug ENDS in a role token (e.g. a stack literally named
|
|
// `my-cache` → stripped to `my`). When the set of actually-deployed stack names is known, cross-reference
|
|
// it so the result is a real stack:
|
|
// - candidate := suffix-strip result.
|
|
// - known[candidate] → candidate (a real DB-role suffix of a real stack, e.g. romm-postgres→romm).
|
|
// - else known[containerName] → containerName (the container name IS the stack — don't strip, e.g. my-cache).
|
|
// - else longest known prefix → handles <stack>_postgres / <stack>-1 / compose-suffixed names.
|
|
// - else → candidate (fall back to today's suffix-strip; preserves behaviour when
|
|
// the stack list is empty/unavailable, so nothing regresses).
|
|
// A nil/empty `known` map = the legacy fast path (pure suffix-strip).
|
|
func deriveStackName(containerName string, known map[string]bool) string {
|
|
candidate := suffixStripStackName(containerName)
|
|
|
|
if len(known) == 0 {
|
|
return candidate
|
|
}
|
|
if known[candidate] {
|
|
return candidate
|
|
}
|
|
if known[containerName] {
|
|
return containerName
|
|
}
|
|
// Longest known stack name that is a prefix of the container name (tie-break: longest wins).
|
|
best := ""
|
|
for name := range known {
|
|
if name == "" || len(name) >= len(containerName) {
|
|
continue
|
|
}
|
|
// boundary char so "rom" doesn't match "romm-..."; compose/role separators are - or _.
|
|
sep := containerName[len(name)]
|
|
if strings.HasPrefix(containerName, name) && (sep == '-' || sep == '_') && len(name) > len(best) {
|
|
best = name
|
|
}
|
|
}
|
|
if best != "" {
|
|
return best
|
|
}
|
|
return candidate
|
|
}
|
|
|
|
// suffixStripStackName is the legacy pure suffix-strip (the M19 fallback when no stack list is known).
|
|
func suffixStripStackName(containerName string) string {
|
|
knownSuffixes := []string{"postgres", "db", "mariadb", "mysql", "database", "redis", "cache"}
|
|
|
|
parts := strings.Split(containerName, "-")
|
|
if len(parts) <= 1 {
|
|
return containerName
|
|
}
|
|
|
|
last := strings.ToLower(parts[len(parts)-1])
|
|
for _, suffix := range knownSuffixes {
|
|
if last == suffix {
|
|
return strings.Join(parts[:len(parts)-1], "-")
|
|
}
|
|
}
|
|
return containerName
|
|
}
|
|
|
|
func cleanupTmpFiles(dumpDir string, logger *log.Logger) {
|
|
entries, err := os.ReadDir(dumpDir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
cutoff := time.Now().Add(-1 * time.Hour)
|
|
for _, e := range entries {
|
|
if !strings.HasSuffix(e.Name(), ".tmp") {
|
|
continue
|
|
}
|
|
info, err := e.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if info.ModTime().Before(cutoff) {
|
|
path := filepath.Join(dumpDir, e.Name())
|
|
os.Remove(path)
|
|
logger.Printf("[INFO] [backup] Cleaned up stale tmp file: %s", e.Name())
|
|
}
|
|
}
|
|
}
|
|
|
|
// M1: formatBytes removed — use humanizeBytes() from appdata.go (same package, no duplication).
|