v0.14.0: Per-drive backup architecture + storage path overhaul
Major refactor of backup and storage paths: - Per-drive restic repos at <drive>/backups/primary/restic/ - Per-app DB dumps at <drive>/backups/primary/<app>/db-dumps/ - Remove global BackupDir, DBDumpDir, ResticRepo config fields - Add SystemDataPath config (fallback for apps without HDD) - New backup/paths.go with pure path computation helpers - Add GetStackHDDPath to StackDataProvider interface - Restic methods now accept repoPath as parameter - Cross-drive backup uses new secondary path structure - Rename storage/ to appdata/ in scripts and compose templates - Update protected HDD paths (storage → appdata + backups) - Simplify backup UI (remove global path displays) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -132,13 +132,12 @@ func main() {
|
||||
}
|
||||
|
||||
// --- Initialize cross-drive backup runner ---
|
||||
crossDriveRunner := backup.NewCrossDriveRunner(sett, stackProv, logger)
|
||||
crossDriveRunner := backup.NewCrossDriveRunner(sett, stackProv, cfg.Paths.SystemDataPath, logger)
|
||||
|
||||
// Wire cross-drive → backup manager for pre-backup DB dumps
|
||||
if backupMgr != nil {
|
||||
crossDriveRunner.SetDBDumper(backupMgr)
|
||||
}
|
||||
crossDriveRunner.SetDBDumpDir(cfg.Paths.DBDumpDir)
|
||||
|
||||
// --- Initialize alert manager ---
|
||||
alertMgr := web.NewAlertManager(logger)
|
||||
@@ -449,6 +448,19 @@ func (a *stackAdapter) GetStackHDDMounts(name string) []string {
|
||||
return allMounts
|
||||
}
|
||||
|
||||
func (a *stackAdapter) GetStackHDDPath(name string) string {
|
||||
s, ok := a.mgr.GetStack(name)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
stackDir := filepath.Dir(s.ComposePath)
|
||||
appCfg := stacks.LoadAppConfig(stackDir)
|
||||
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
|
||||
return filepath.Clean(appCfg.Env["HDD_PATH"])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// discoverHDDPaths scans deployed apps' app.yaml for HDD_PATH env values.
|
||||
func discoverHDDPaths(stacksDir string, logger *log.Logger) []string {
|
||||
entries, err := os.ReadDir(stacksDir)
|
||||
|
||||
@@ -29,8 +29,7 @@ infrastructure:
|
||||
paths:
|
||||
stacks_dir: "/opt/docker/stacks" # Where compose files live
|
||||
data_dir: "/opt/docker/felhom-controller/data"
|
||||
backup_dir: "/srv/backups"
|
||||
db_dump_dir: "/srv/backups/db-dumps"
|
||||
system_data_path: "/mnt/sys_drive" # NVMe/system drive mount — fallback for apps without HDD
|
||||
hdd_path: "" # DEPRECATED: use Settings > Adattárolók instead. Fallback only for auto-discovery.
|
||||
|
||||
# --- System ---
|
||||
@@ -63,9 +62,12 @@ stacks:
|
||||
compose_command: ""
|
||||
|
||||
# --- Backup ---
|
||||
# Per-drive backup paths are computed automatically:
|
||||
# <drive>/backups/primary/restic/ — restic repo per drive
|
||||
# <drive>/backups/primary/<app>/db-dumps/ — DB dumps per app
|
||||
# <drive>/backups/secondary/ — cross-drive rsync + restic
|
||||
backup:
|
||||
enabled: true
|
||||
restic_repo: "/srv/backups/restic-repo"
|
||||
restic_password_file: "/opt/docker/felhom-controller/data/restic-password"
|
||||
db_dump_schedule: "02:30"
|
||||
restic_schedule: "03:00"
|
||||
|
||||
@@ -16,6 +16,7 @@ type StackDataProvider interface {
|
||||
GetStackComposePath(name string) (composePath string, ok bool)
|
||||
ListDeployedStacks() []StackSummary
|
||||
GetStackHDDMounts(name string) []string
|
||||
GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD)
|
||||
StopStack(name string) error
|
||||
StartStack(name string) error
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -17,12 +18,13 @@ import (
|
||||
|
||||
// Manager orchestrates database dumps and restic backups.
|
||||
type Manager struct {
|
||||
cfg *config.Config
|
||||
restic *ResticManager
|
||||
logger *log.Logger
|
||||
pinger *monitor.Pinger
|
||||
settings *settings.Settings
|
||||
stackProvider StackDataProvider
|
||||
cfg *config.Config
|
||||
restic *ResticManager
|
||||
logger *log.Logger
|
||||
pinger *monitor.Pinger
|
||||
settings *settings.Settings
|
||||
stackProvider StackDataProvider
|
||||
systemDataPath string // fallback drive for SSD-only apps
|
||||
|
||||
mu sync.Mutex
|
||||
lastDBDump *DBDumpStatus
|
||||
@@ -92,8 +94,6 @@ type FullBackupStatus struct {
|
||||
Retention config.RetentionConfig
|
||||
|
||||
// Repository health
|
||||
RepoPath string
|
||||
BackupPaths []string
|
||||
LastCheckTime time.Time
|
||||
LastCheckOK bool
|
||||
|
||||
@@ -133,15 +133,51 @@ type BackupStatus struct {
|
||||
// NewManager creates a new backup manager.
|
||||
func NewManager(cfg *config.Config, pinger *monitor.Pinger, sett *settings.Settings, logger *log.Logger) *Manager {
|
||||
return &Manager{
|
||||
cfg: cfg,
|
||||
restic: NewResticManager(cfg, logger),
|
||||
logger: logger,
|
||||
pinger: pinger,
|
||||
settings: sett,
|
||||
cfg: cfg,
|
||||
restic: NewResticManager(cfg, logger),
|
||||
logger: logger,
|
||||
pinger: pinger,
|
||||
settings: sett,
|
||||
systemDataPath: cfg.Paths.SystemDataPath,
|
||||
}
|
||||
}
|
||||
|
||||
// RunDBDumps discovers and dumps all databases.
|
||||
// GetAppDrivePath returns the drive path for an app.
|
||||
// Uses HDD_PATH from app.yaml if set, otherwise falls back to system data path.
|
||||
func (m *Manager) GetAppDrivePath(stackName string) string {
|
||||
if m.stackProvider != nil {
|
||||
if hddPath := m.stackProvider.GetStackHDDPath(stackName); hddPath != "" {
|
||||
return hddPath
|
||||
}
|
||||
}
|
||||
return m.systemDataPath
|
||||
}
|
||||
|
||||
// groupStacksByDrive groups deployed stacks by their home drive path.
|
||||
func (m *Manager) groupStacksByDrive() map[string][]StackSummary {
|
||||
if m.stackProvider == nil {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string][]StackSummary)
|
||||
for _, stack := range m.stackProvider.ListDeployedStacks() {
|
||||
drive := m.GetAppDrivePath(stack.Name)
|
||||
result[drive] = append(result[drive], stack)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// activeDrives returns sorted list of drives that have deployed apps.
|
||||
func (m *Manager) activeDrives() []string {
|
||||
groups := m.groupStacksByDrive()
|
||||
var drives []string
|
||||
for d := range groups {
|
||||
drives = append(drives, d)
|
||||
}
|
||||
sort.Strings(drives)
|
||||
return drives
|
||||
}
|
||||
|
||||
// RunDBDumps discovers and dumps all databases to per-drive, per-app paths.
|
||||
func (m *Manager) RunDBDumps(ctx context.Context) error {
|
||||
start := time.Now()
|
||||
m.logger.Printf("[INFO] Starting database dump run")
|
||||
@@ -166,31 +202,37 @@ func (m *Manager) RunDBDumps(ctx context.Context) error {
|
||||
|
||||
m.logger.Printf("[INFO] Discovered %d database(s): %s", len(dbs), dbNames(dbs))
|
||||
|
||||
results := DumpAll(ctx, dbs, m.cfg.Paths.DBDumpDir, m.logger)
|
||||
|
||||
// Check results and persist validations
|
||||
// Dump each DB to its app's drive path
|
||||
var results []DumpResult
|
||||
allOK := true
|
||||
var summary []string
|
||||
var totalSize int64
|
||||
for _, r := range results {
|
||||
if r.Error != nil {
|
||||
|
||||
for _, db := range dbs {
|
||||
drivePath := m.GetAppDrivePath(db.StackName)
|
||||
dumpDir := AppDBDumpPath(drivePath, db.StackName)
|
||||
|
||||
result := DumpOne(ctx, db, dumpDir, m.logger)
|
||||
results = append(results, result)
|
||||
|
||||
if result.Error != nil {
|
||||
allOK = false
|
||||
summary = append(summary, fmt.Sprintf("FAIL %s: %v", r.DB.ContainerName, r.Error))
|
||||
m.logger.Printf("[ERROR] DB dump failed for %s: %v", r.DB.ContainerName, r.Error)
|
||||
summary = append(summary, fmt.Sprintf("FAIL %s: %v", result.DB.ContainerName, result.Error))
|
||||
m.logger.Printf("[ERROR] DB dump failed for %s: %v", result.DB.ContainerName, result.Error)
|
||||
} else {
|
||||
totalSize += r.Size
|
||||
summary = append(summary, fmt.Sprintf("OK %s (%s)", r.DB.ContainerName, humanizeBytes(r.Size)))
|
||||
totalSize += result.Size
|
||||
summary = append(summary, fmt.Sprintf("OK %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size)))
|
||||
|
||||
// Persist validation result to settings.json
|
||||
if m.settings != nil && r.FilePath != "" {
|
||||
filename := filepath.Base(r.FilePath)
|
||||
if m.settings != nil && result.FilePath != "" {
|
||||
filename := filepath.Base(result.FilePath)
|
||||
cache := settings.DBValidationCache{
|
||||
ValidatedAt: time.Now().Format(time.RFC3339),
|
||||
TableCount: r.Validation.TableCount,
|
||||
HasHeader: r.Validation.Valid,
|
||||
TableCount: result.Validation.TableCount,
|
||||
HasHeader: result.Validation.Valid,
|
||||
}
|
||||
if !r.Validation.Valid {
|
||||
cache.Error = r.Validation.Error
|
||||
if !result.Validation.Valid {
|
||||
cache.Error = result.Validation.Error
|
||||
}
|
||||
if err := m.settings.SetDBValidation(filename, cache); err != nil {
|
||||
m.logger.Printf("[WARN] Failed to cache validation for %s: %v", filename, err)
|
||||
@@ -226,132 +268,185 @@ func (m *Manager) RunDBDumps(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunBackup runs a restic backup snapshot.
|
||||
// RunBackup runs per-drive restic backup snapshots.
|
||||
func (m *Manager) RunBackup(ctx context.Context) error {
|
||||
start := time.Now()
|
||||
m.logger.Printf("[INFO] Starting restic backup")
|
||||
m.logger.Printf("[INFO] Starting restic backup (per-drive)")
|
||||
|
||||
// Ensure repo is initialized
|
||||
if err := m.restic.EnsureInitialized(); err != nil {
|
||||
m.logger.Printf("[ERROR] Restic init failed: %v", err)
|
||||
m.pinger.Fail(m.cfg.Monitoring.PingUUIDs.Backup, fmt.Sprintf("Restic init failed: %v", err))
|
||||
return err
|
||||
driveStacks := m.groupStacksByDrive()
|
||||
if len(driveStacks) == 0 {
|
||||
m.logger.Printf("[INFO] No deployed stacks — skipping backup")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Backup paths: base + dynamic app data
|
||||
paths := []string{
|
||||
// Infrastructure paths included in every drive's primary repo
|
||||
infraPaths := []string{
|
||||
m.cfg.Paths.StacksDir,
|
||||
m.cfg.Paths.DBDumpDir,
|
||||
"/opt/docker/felhom-controller/controller.yaml",
|
||||
}
|
||||
appPaths := m.resolveAppBackupPaths()
|
||||
if len(appPaths) > 0 {
|
||||
paths = append(paths, appPaths...)
|
||||
m.logger.Printf("[INFO] Backup paths (%d total, %d app data): %v", len(paths), len(appPaths), paths)
|
||||
|
||||
var lastResult *SnapshotResult
|
||||
var anyErr error
|
||||
driveCount := 0
|
||||
|
||||
for drivePath, stacks := range driveStacks {
|
||||
repoPath := PrimaryResticRepoPath(drivePath)
|
||||
|
||||
// Ensure repo is initialized
|
||||
if err := m.restic.EnsureInitialized(repoPath); err != nil {
|
||||
m.logger.Printf("[ERROR] Restic init failed for %s: %v", repoPath, err)
|
||||
anyErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
// Build paths for this drive
|
||||
var paths []string
|
||||
paths = append(paths, infraPaths...)
|
||||
|
||||
for _, stack := range stacks {
|
||||
// App data (appdata/<stack>/)
|
||||
appData := AppDataDir(drivePath, stack.Name)
|
||||
if _, err := os.Stat(appData); err == nil {
|
||||
paths = append(paths, appData)
|
||||
}
|
||||
// HDD mounts (for apps with custom mount points)
|
||||
if m.stackProvider != nil {
|
||||
for _, mount := range m.stackProvider.GetStackHDDMounts(stack.Name) {
|
||||
if _, err := os.Stat(mount); err == nil {
|
||||
paths = append(paths, mount)
|
||||
}
|
||||
}
|
||||
}
|
||||
// DB dumps for this stack
|
||||
dumpDir := AppDBDumpPath(drivePath, stack.Name)
|
||||
if _, err := os.Stat(dumpDir); err == nil {
|
||||
paths = append(paths, dumpDir)
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate paths
|
||||
paths = dedup(paths)
|
||||
|
||||
tags := []string{"felhom", m.cfg.Customer.ID, filepath.Base(drivePath)}
|
||||
m.logger.Printf("[INFO] Backing up drive %s (%d apps, %d paths)", drivePath, len(stacks), len(paths))
|
||||
|
||||
result, err := m.restic.Snapshot(repoPath, paths, tags)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERROR] Restic backup failed for drive %s: %v", drivePath, err)
|
||||
anyErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
lastResult = result
|
||||
driveCount++
|
||||
|
||||
// Prune check (weekly — Sunday)
|
||||
if shouldPrune(m.cfg.Backup.PruneSchedule) {
|
||||
m.logger.Printf("[INFO] Running weekly prune for %s", repoPath)
|
||||
if err := m.restic.Prune(repoPath, m.cfg.Backup.Retention); err != nil {
|
||||
m.logger.Printf("[WARN] Restic prune failed for %s: %v", repoPath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
tags := []string{"felhom", m.cfg.Customer.ID}
|
||||
|
||||
result, err := m.restic.Snapshot(paths, tags)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERROR] Restic backup failed: %v", err)
|
||||
m.pinger.Fail(m.cfg.Monitoring.PingUUIDs.Backup, fmt.Sprintf("Backup failed: %v", err))
|
||||
duration := time.Since(start)
|
||||
|
||||
if anyErr != nil && driveCount == 0 {
|
||||
// All drives failed
|
||||
m.pinger.Fail(m.cfg.Monitoring.PingUUIDs.Backup, fmt.Sprintf("Backup failed: %v", anyErr))
|
||||
m.mu.Lock()
|
||||
m.lastBackup = &BackupStatus{
|
||||
LastRun: time.Now(),
|
||||
Success: false,
|
||||
Duration: time.Since(start),
|
||||
Duration: duration,
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return err
|
||||
return anyErr
|
||||
}
|
||||
|
||||
// Prune check (weekly — Sunday)
|
||||
if shouldPrune(m.cfg.Backup.PruneSchedule) {
|
||||
m.logger.Printf("[INFO] Running weekly prune")
|
||||
if err := m.restic.Prune(m.cfg.Backup.Retention); err != nil {
|
||||
m.logger.Printf("[WARN] Restic prune failed: %v", err)
|
||||
}
|
||||
checkErr := m.restic.Check()
|
||||
if checkErr != nil {
|
||||
m.logger.Printf("[WARN] Restic check failed: %v", checkErr)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.lastCheckTime = time.Now()
|
||||
m.lastCheckOK = checkErr == nil
|
||||
m.mu.Unlock()
|
||||
}
|
||||
// Get aggregated stats
|
||||
stats := m.aggregateRepoStats()
|
||||
|
||||
// Get stats
|
||||
stats, _ := m.restic.Stats()
|
||||
|
||||
duration := time.Since(start)
|
||||
m.mu.Lock()
|
||||
m.lastBackup = &BackupStatus{
|
||||
LastRun: time.Now(),
|
||||
Snapshot: result,
|
||||
Success: true,
|
||||
Snapshot: lastResult,
|
||||
Success: anyErr == nil,
|
||||
Duration: duration,
|
||||
RepoStats: stats,
|
||||
}
|
||||
// Append to snapshot history
|
||||
m.appendSnapshotRecord(SnapshotRecord{
|
||||
SnapshotID: result.SnapshotID,
|
||||
Time: time.Now(),
|
||||
FilesNew: result.FilesNew,
|
||||
FilesChanged: result.FilesChanged,
|
||||
DataAdded: result.DataAdded,
|
||||
Duration: duration,
|
||||
Success: true,
|
||||
HasStats: true,
|
||||
})
|
||||
if lastResult != nil {
|
||||
m.appendSnapshotRecord(SnapshotRecord{
|
||||
SnapshotID: lastResult.SnapshotID,
|
||||
Time: time.Now(),
|
||||
FilesNew: lastResult.FilesNew,
|
||||
FilesChanged: lastResult.FilesChanged,
|
||||
DataAdded: lastResult.DataAdded,
|
||||
Duration: duration,
|
||||
Success: true,
|
||||
HasStats: true,
|
||||
})
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
body := fmt.Sprintf("Backup OK\nSnapshot: %s\nNew files: %d, Changed: %d\nData added: %s\nDuration: %s",
|
||||
result.SnapshotID, result.FilesNew, result.FilesChanged, result.DataAdded,
|
||||
duration.Round(time.Second))
|
||||
m.pinger.Ping(m.cfg.Monitoring.PingUUIDs.Backup, body)
|
||||
if lastResult != nil {
|
||||
body := fmt.Sprintf("Backup OK (%d drives)\nSnapshot: %s\nNew files: %d, Changed: %d\nData added: %s\nDuration: %s",
|
||||
driveCount, lastResult.SnapshotID, lastResult.FilesNew, lastResult.FilesChanged, lastResult.DataAdded,
|
||||
duration.Round(time.Second))
|
||||
m.pinger.Ping(m.cfg.Monitoring.PingUUIDs.Backup, body)
|
||||
|
||||
m.logger.Printf("[INFO] Restic backup completed: snapshot %s, %d new, %d changed, %s added (%s)",
|
||||
result.SnapshotID, result.FilesNew, result.FilesChanged, result.DataAdded,
|
||||
duration.Round(time.Millisecond))
|
||||
m.logger.Printf("[INFO] Restic backup completed: %d drives, snapshot %s, %d new, %d changed, %s added (%s)",
|
||||
driveCount, lastResult.SnapshotID, lastResult.FilesNew, lastResult.FilesChanged, lastResult.DataAdded,
|
||||
duration.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
// Refresh cache so the page shows updated data immediately
|
||||
if m.AfterBackup != nil {
|
||||
m.AfterBackup()
|
||||
}
|
||||
|
||||
return nil
|
||||
return anyErr
|
||||
}
|
||||
|
||||
// RunIntegrityCheck runs restic check and pings healthchecks with the result.
|
||||
// RunIntegrityCheck runs restic check on all primary repos and pings healthchecks.
|
||||
func (m *Manager) RunIntegrityCheck(ctx context.Context) error {
|
||||
m.logger.Printf("[INFO] Starting restic integrity check")
|
||||
start := time.Now()
|
||||
|
||||
if err := m.restic.EnsureInitialized(); err != nil {
|
||||
m.logger.Printf("[ERROR] Restic init failed for integrity check: %v", err)
|
||||
return err
|
||||
drives := m.activeDrives()
|
||||
if len(drives) == 0 {
|
||||
m.logger.Printf("[INFO] No active drives — skipping integrity check")
|
||||
return nil
|
||||
}
|
||||
|
||||
err := m.restic.Check()
|
||||
duration := time.Since(start)
|
||||
var checkErr error
|
||||
for _, drive := range drives {
|
||||
repoPath := PrimaryResticRepoPath(drive)
|
||||
if !m.restic.RepoExists(repoPath) {
|
||||
continue
|
||||
}
|
||||
if err := m.restic.Check(repoPath); err != nil {
|
||||
m.logger.Printf("[ERROR] Restic check failed for %s: %v", repoPath, err)
|
||||
checkErr = err
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(start)
|
||||
uuid := m.cfg.Monitoring.PingUUIDs.BackupIntegrity
|
||||
|
||||
m.mu.Lock()
|
||||
m.lastCheckTime = time.Now()
|
||||
m.lastCheckOK = err == nil
|
||||
m.lastCheckOK = checkErr == nil
|
||||
m.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERROR] Restic integrity check failed (%s): %v", duration.Round(time.Second), err)
|
||||
m.pinger.Fail(uuid, fmt.Sprintf("restic check failed: %v", err))
|
||||
return err
|
||||
if checkErr != nil {
|
||||
m.logger.Printf("[ERROR] Restic integrity check failed (%s): %v", duration.Round(time.Second), checkErr)
|
||||
m.pinger.Fail(uuid, fmt.Sprintf("restic check failed: %v", checkErr))
|
||||
return checkErr
|
||||
}
|
||||
|
||||
m.logger.Printf("[INFO] Restic integrity check passed (%s)", duration.Round(time.Second))
|
||||
m.pinger.Ping(uuid, fmt.Sprintf("restic check passed (%s)", duration.Round(time.Second)))
|
||||
m.logger.Printf("[INFO] Restic integrity check passed (%d repos, %s)", len(drives), duration.Round(time.Second))
|
||||
m.pinger.Ping(uuid, fmt.Sprintf("restic check passed (%d repos, %s)", len(drives), duration.Round(time.Second)))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -387,9 +482,13 @@ func (m *Manager) GetStatus() (*DBDumpStatus, *BackupStatus) {
|
||||
return m.lastDBDump, m.lastBackup
|
||||
}
|
||||
|
||||
// GetRepoStats returns repository statistics.
|
||||
// GetRepoStats returns aggregated repository statistics across all primary repos.
|
||||
func (m *Manager) GetRepoStats() (*RepoStats, error) {
|
||||
return m.restic.Stats()
|
||||
stats := m.aggregateRepoStats()
|
||||
if stats.SnapshotCount == 0 && stats.TotalSize == "" {
|
||||
return stats, fmt.Errorf("no repos available")
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// IsRunning returns whether a backup or restore is currently in progress.
|
||||
@@ -404,9 +503,33 @@ func (m *Manager) GetResticPassword() (string, error) {
|
||||
return m.restic.GetPassword()
|
||||
}
|
||||
|
||||
// ListSnapshots returns snapshots from the restic repository.
|
||||
// ListSnapshots returns snapshots from all primary restic repositories, merged and sorted.
|
||||
func (m *Manager) ListSnapshots(limit int) ([]SnapshotInfo, error) {
|
||||
return m.restic.ListSnapshots(limit)
|
||||
drives := m.activeDrives()
|
||||
var allSnapshots []SnapshotInfo
|
||||
for _, drive := range drives {
|
||||
repoPath := PrimaryResticRepoPath(drive)
|
||||
if !m.restic.RepoExists(repoPath) {
|
||||
continue
|
||||
}
|
||||
snapshots, err := m.restic.ListSnapshots(repoPath, 0)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] Could not list snapshots from %s: %v", repoPath, err)
|
||||
continue
|
||||
}
|
||||
for i := range snapshots {
|
||||
snapshots[i].RepoPath = repoPath
|
||||
}
|
||||
allSnapshots = append(allSnapshots, snapshots...)
|
||||
}
|
||||
// Sort newest first
|
||||
sort.Slice(allSnapshots, func(i, j int) bool {
|
||||
return allSnapshots[i].Time.After(allSnapshots[j].Time)
|
||||
})
|
||||
if limit > 0 && len(allSnapshots) > limit {
|
||||
allSnapshots = allSnapshots[:limit]
|
||||
}
|
||||
return allSnapshots, nil
|
||||
}
|
||||
|
||||
// SetStackProvider sets the stack data provider for app data discovery.
|
||||
@@ -425,34 +548,8 @@ func (m *Manager) GetStackHDDMounts(name string) []string {
|
||||
return m.stackProvider.GetStackHDDMounts(name)
|
||||
}
|
||||
|
||||
// resolveAppBackupPaths returns HDD paths for ALL deployed apps.
|
||||
// User data backup is mandatory — every app with HDD mounts is included.
|
||||
func (m *Manager) resolveAppBackupPaths() []string {
|
||||
if m.stackProvider == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var paths []string
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, stack := range m.stackProvider.ListDeployedStacks() {
|
||||
hddMounts := m.stackProvider.GetStackHDDMounts(stack.Name)
|
||||
for _, mount := range hddMounts {
|
||||
if seen[mount] {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(mount); err == nil {
|
||||
paths = append(paths, mount)
|
||||
seen[mount] = true
|
||||
m.logger.Printf("[DEBUG] Including app data: %s (from %s)", mount, stack.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
// DumpStackDB runs a database dump for containers belonging to a specific stack.
|
||||
// Used by cross-drive backup to ensure DB state matches user data.
|
||||
// Dumps to the stack's home drive: <drive>/backups/primary/<stack>/db-dumps/.
|
||||
func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
|
||||
dbs, err := DiscoverDatabases(ctx, m.logger)
|
||||
if err != nil {
|
||||
@@ -470,25 +567,28 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
m.logger.Printf("[INFO] Running pre-backup DB dump for %s (%d database(s))", stackName, len(stackDBs))
|
||||
results := DumpAll(ctx, stackDBs, m.cfg.Paths.DBDumpDir, m.logger)
|
||||
drivePath := m.GetAppDrivePath(stackName)
|
||||
dumpDir := AppDBDumpPath(drivePath, stackName)
|
||||
|
||||
for _, r := range results {
|
||||
if r.Error != nil {
|
||||
return fmt.Errorf("DB dump failed for %s: %w", r.DB.ContainerName, r.Error)
|
||||
m.logger.Printf("[INFO] Running pre-backup DB dump for %s (%d database(s)) → %s", stackName, len(stackDBs), dumpDir)
|
||||
|
||||
for _, db := range stackDBs {
|
||||
result := DumpOne(ctx, db, dumpDir, m.logger)
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("DB dump failed for %s: %w", result.DB.ContainerName, result.Error)
|
||||
}
|
||||
m.logger.Printf("[INFO] Pre-backup DB dump OK: %s (%s)", r.DB.ContainerName, humanizeBytes(r.Size))
|
||||
m.logger.Printf("[INFO] Pre-backup DB dump OK: %s (%s)", result.DB.ContainerName, humanizeBytes(result.Size))
|
||||
|
||||
// Persist validation to settings
|
||||
if m.settings != nil && r.FilePath != "" {
|
||||
filename := filepath.Base(r.FilePath)
|
||||
if m.settings != nil && result.FilePath != "" {
|
||||
filename := filepath.Base(result.FilePath)
|
||||
cache := settings.DBValidationCache{
|
||||
ValidatedAt: time.Now().Format(time.RFC3339),
|
||||
TableCount: r.Validation.TableCount,
|
||||
HasHeader: r.Validation.Valid,
|
||||
TableCount: result.Validation.TableCount,
|
||||
HasHeader: result.Validation.Valid,
|
||||
}
|
||||
if !r.Validation.Valid {
|
||||
cache.Error = r.Validation.Error
|
||||
if !result.Validation.Valid {
|
||||
cache.Error = result.Validation.Error
|
||||
}
|
||||
_ = m.settings.SetDBValidation(filename, cache)
|
||||
}
|
||||
@@ -496,6 +596,51 @@ func (m *Manager) DumpStackDB(ctx context.Context, stackName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// aggregateRepoStats combines stats from all primary restic repos.
|
||||
func (m *Manager) aggregateRepoStats() *RepoStats {
|
||||
drives := m.activeDrives()
|
||||
agg := &RepoStats{}
|
||||
var totalBytes int64
|
||||
|
||||
for _, drive := range drives {
|
||||
repoPath := PrimaryResticRepoPath(drive)
|
||||
if !m.restic.RepoExists(repoPath) {
|
||||
continue
|
||||
}
|
||||
stats, err := m.restic.Stats(repoPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
agg.SnapshotCount += stats.SnapshotCount
|
||||
totalBytes += stats.TotalSizeBytes
|
||||
if stats.LatestSnapshot != nil {
|
||||
if agg.LatestSnapshot == nil || stats.LatestSnapshot.Time.After(agg.LatestSnapshot.Time) {
|
||||
agg.LatestSnapshot = stats.LatestSnapshot
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
agg.TotalSizeBytes = totalBytes
|
||||
if totalBytes > 0 {
|
||||
agg.TotalSize = humanizeBytes(totalBytes)
|
||||
}
|
||||
return agg
|
||||
}
|
||||
|
||||
// listAllDumpFiles scans per-drive per-stack DB dump directories.
|
||||
func (m *Manager) listAllDumpFiles() []DumpFileInfo {
|
||||
var allFiles []DumpFileInfo
|
||||
for drive, stacks := range m.groupStacksByDrive() {
|
||||
for _, stack := range stacks {
|
||||
dumpDir := AppDBDumpPath(drive, stack.Name)
|
||||
if files, err := ListDumpFiles(dumpDir); err == nil {
|
||||
allFiles = append(allFiles, files...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return allFiles
|
||||
}
|
||||
|
||||
func shouldPrune(schedule string) bool {
|
||||
loc, err := time.LoadLocation("Europe/Budapest")
|
||||
if err != nil {
|
||||
@@ -521,18 +666,33 @@ func (m *Manager) appendSnapshotRecord(rec SnapshotRecord) {
|
||||
}
|
||||
}
|
||||
|
||||
// LoadSnapshotHistory populates the snapshot history from restic on startup.
|
||||
// LoadSnapshotHistory populates the snapshot history from all primary restic repos on startup.
|
||||
func (m *Manager) LoadSnapshotHistory() {
|
||||
snapshots, err := m.restic.ListSnapshots(20)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] Could not load snapshot history: %v", err)
|
||||
return
|
||||
drives := m.activeDrives()
|
||||
var allSnapshots []SnapshotInfo
|
||||
|
||||
for _, drive := range drives {
|
||||
repoPath := PrimaryResticRepoPath(drive)
|
||||
if !m.restic.RepoExists(repoPath) {
|
||||
continue
|
||||
}
|
||||
snapshots, err := m.restic.ListSnapshots(repoPath, 20)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] Could not load snapshot history from %s: %v", repoPath, err)
|
||||
continue
|
||||
}
|
||||
allSnapshots = append(allSnapshots, snapshots...)
|
||||
}
|
||||
|
||||
// Sort by time (oldest first for ring buffer)
|
||||
sort.Slice(allSnapshots, func(i, j int) bool {
|
||||
return allSnapshots[i].Time.Before(allSnapshots[j].Time)
|
||||
})
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for _, s := range snapshots {
|
||||
for _, s := range allSnapshots {
|
||||
m.snapshotHistory = append(m.snapshotHistory, SnapshotRecord{
|
||||
SnapshotID: s.ID,
|
||||
Time: s.Time,
|
||||
@@ -543,7 +703,7 @@ func (m *Manager) LoadSnapshotHistory() {
|
||||
if len(m.snapshotHistory) > 20 {
|
||||
m.snapshotHistory = m.snapshotHistory[len(m.snapshotHistory)-20:]
|
||||
}
|
||||
m.logger.Printf("[INFO] Loaded %d historical snapshots", len(m.snapshotHistory))
|
||||
m.logger.Printf("[INFO] Loaded %d historical snapshots from %d repos", len(m.snapshotHistory), len(drives))
|
||||
}
|
||||
|
||||
// RefreshCache updates the cached full status. Called by scheduler every 5 minutes
|
||||
@@ -558,23 +718,15 @@ func (m *Manager) RefreshCache(nextDBDump, nextBackup time.Time) {
|
||||
NextDBDump: nextDBDump,
|
||||
NextBackup: nextBackup,
|
||||
Retention: m.cfg.Backup.Retention,
|
||||
|
||||
RepoPath: m.cfg.Backup.ResticRepo,
|
||||
BackupPaths: []string{
|
||||
m.cfg.Paths.StacksDir,
|
||||
m.cfg.Paths.DBDumpDir,
|
||||
"/opt/docker/felhom-controller/controller.yaml",
|
||||
},
|
||||
}
|
||||
|
||||
// Expensive calls (outside lock)
|
||||
if stats, err := m.restic.Stats(); err == nil {
|
||||
status.RepoStats = stats
|
||||
}
|
||||
files, filesErr := ListDumpFiles(m.cfg.Paths.DBDumpDir)
|
||||
if filesErr == nil {
|
||||
status.DumpFiles = files
|
||||
}
|
||||
status.RepoStats = m.aggregateRepoStats()
|
||||
|
||||
// Scan dump files from per-drive per-stack paths
|
||||
files := m.listAllDumpFiles()
|
||||
status.DumpFiles = files
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if dbs, err := DiscoverDatabases(ctx, m.logger); err == nil {
|
||||
@@ -584,12 +736,6 @@ func (m *Manager) RefreshCache(nextDBDump, nextBackup time.Time) {
|
||||
// Discover app data — all deployed stacks, backup is mandatory
|
||||
if m.stackProvider != nil {
|
||||
status.AppDataInfo = DiscoverAppData(m.stackProvider, status.DiscoveredDBs)
|
||||
|
||||
// Include enabled app backup paths in the displayed BackupPaths
|
||||
appPaths := m.resolveAppBackupPaths()
|
||||
if len(appPaths) > 0 {
|
||||
status.BackupPaths = append(status.BackupPaths, appPaths...)
|
||||
}
|
||||
}
|
||||
|
||||
// Fill in dynamic fields under lock.
|
||||
@@ -605,7 +751,7 @@ func (m *Manager) RefreshCache(nextDBDump, nextBackup time.Time) {
|
||||
copy(status.SnapshotHistory, m.snapshotHistory)
|
||||
|
||||
// C1: Cross-check lastDBDump results inside lock to prevent torn writes.
|
||||
if m.lastDBDump != nil && filesErr == nil {
|
||||
if m.lastDBDump != nil && len(files) > 0 {
|
||||
fileValidation := make(map[string]DumpValidation) // keyed by filename
|
||||
for _, f := range files {
|
||||
fileValidation[f.FileName] = f.Validation
|
||||
@@ -728,14 +874,8 @@ func (m *Manager) GetFullStatus(nextDBDump, nextBackup time.Time) *FullBackupSta
|
||||
NextDBDump: nextDBDump,
|
||||
NextBackup: nextBackup,
|
||||
Retention: m.cfg.Backup.Retention,
|
||||
RepoPath: m.cfg.Backup.ResticRepo,
|
||||
LastCheckTime: m.lastCheckTime,
|
||||
LastCheckOK: m.lastCheckOK,
|
||||
BackupPaths: []string{
|
||||
m.cfg.Paths.StacksDir,
|
||||
m.cfg.Paths.DBDumpDir,
|
||||
"/opt/docker/felhom-controller/controller.yaml",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -746,3 +886,16 @@ func dbNames(dbs []DiscoveredDB) string {
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// dedup removes duplicate strings from a slice, preserving order.
|
||||
func dedup(items []string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
for _, item := range items {
|
||||
if !seen[item] {
|
||||
seen[item] = true
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -22,22 +22,23 @@ type DBDumper interface {
|
||||
|
||||
// CrossDriveRunner handles per-app backup to secondary storage.
|
||||
type CrossDriveRunner struct {
|
||||
sett *settings.Settings
|
||||
stackProvider StackDataProvider
|
||||
dbDumper DBDumper
|
||||
dbDumpDir string // path to DB dump directory (e.g., /srv/backups/db-dumps)
|
||||
logger *log.Logger
|
||||
mu sync.Mutex
|
||||
running map[string]bool // per-app running state
|
||||
sett *settings.Settings
|
||||
stackProvider StackDataProvider
|
||||
dbDumper DBDumper
|
||||
systemDataPath string // fallback drive for SSD-only apps
|
||||
logger *log.Logger
|
||||
mu sync.Mutex
|
||||
running map[string]bool // per-app running state
|
||||
}
|
||||
|
||||
// NewCrossDriveRunner creates a new CrossDriveRunner.
|
||||
func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, logger *log.Logger) *CrossDriveRunner {
|
||||
func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, systemDataPath string, logger *log.Logger) *CrossDriveRunner {
|
||||
return &CrossDriveRunner{
|
||||
sett: sett,
|
||||
stackProvider: provider,
|
||||
logger: logger,
|
||||
running: make(map[string]bool),
|
||||
sett: sett,
|
||||
stackProvider: provider,
|
||||
systemDataPath: systemDataPath,
|
||||
logger: logger,
|
||||
running: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,9 +48,12 @@ func (r *CrossDriveRunner) SetDBDumper(d DBDumper) {
|
||||
r.dbDumper = d
|
||||
}
|
||||
|
||||
// SetDBDumpDir sets the path to the DB dump directory for cross-drive backups.
|
||||
func (r *CrossDriveRunner) SetDBDumpDir(dir string) {
|
||||
r.dbDumpDir = dir
|
||||
// getAppDrivePath returns the drive path for an app.
|
||||
func (r *CrossDriveRunner) getAppDrivePath(stackName string) string {
|
||||
if hddPath := r.stackProvider.GetStackHDDPath(stackName); hddPath != "" {
|
||||
return hddPath
|
||||
}
|
||||
return r.systemDataPath
|
||||
}
|
||||
|
||||
// RunAppBackup runs cross-drive backup for a single app.
|
||||
@@ -128,7 +132,7 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e
|
||||
// Calculate backup size
|
||||
var sizeHuman string
|
||||
if cfg.Method == "rsync" {
|
||||
destDir := filepath.Join(cfg.DestinationPath, "backups", "rsync", stackName)
|
||||
destDir := AppSecondaryRsyncPath(cfg.DestinationPath, stackName)
|
||||
if sz, err := dirSizeBytes(destDir); err == nil {
|
||||
sizeHuman = humanizeBytes(sz)
|
||||
}
|
||||
@@ -220,7 +224,7 @@ func (r *CrossDriveRunner) ValidateDestination(path string) error {
|
||||
// --- rsync ---
|
||||
|
||||
func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBase string, mounts []string) error {
|
||||
destDir := filepath.Join(destBase, "backups", "rsync", stackName)
|
||||
destDir := AppSecondaryRsyncPath(destBase, stackName)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating rsync dest dir: %w", err)
|
||||
}
|
||||
@@ -261,7 +265,7 @@ func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBa
|
||||
}
|
||||
}
|
||||
|
||||
// --- Copy DB dumps for this stack ---
|
||||
// --- Copy DB dumps for this stack from its home drive ---
|
||||
dbDestDir := filepath.Join(destDir, "_db")
|
||||
if err := os.MkdirAll(dbDestDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating DB dump dest dir: %w", err)
|
||||
@@ -294,7 +298,7 @@ func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBa
|
||||
// --- restic ---
|
||||
|
||||
func (r *CrossDriveRunner) runResticBackup(ctx context.Context, stackName, destBase string, mounts []string) error {
|
||||
repoPath := filepath.Join(destBase, "backups", "restic")
|
||||
repoPath := SecondaryResticRepoPath(destBase)
|
||||
|
||||
// Get or create the cross-drive restic password
|
||||
password, err := r.sett.GetOrCreateCrossDrivePassword()
|
||||
@@ -334,11 +338,11 @@ func (r *CrossDriveRunner) runResticBackup(ctx context.Context, stackName, destB
|
||||
if composePath, ok := r.stackProvider.GetStackComposePath(stackName); ok {
|
||||
args = append(args, filepath.Dir(composePath))
|
||||
}
|
||||
// Include DB dump dir (all stacks' dumps — restic deduplicates)
|
||||
if r.dbDumpDir != "" {
|
||||
if _, err := os.Stat(r.dbDumpDir); err == nil {
|
||||
args = append(args, r.dbDumpDir)
|
||||
}
|
||||
// Include DB dump dir for this app (from its home drive)
|
||||
appDrive := r.getAppDrivePath(stackName)
|
||||
dumpDir := AppDBDumpPath(appDrive, stackName)
|
||||
if _, err := os.Stat(dumpDir); err == nil {
|
||||
args = append(args, dumpDir)
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "restic", args...)
|
||||
@@ -387,27 +391,26 @@ func (r *CrossDriveRunner) ensureResticRepo(ctx context.Context, repoPath, pwFil
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyStackDBDumps copies DB dump files for the given stack to destDir.
|
||||
// DB dump files are named <stackName>_<dbtype>.sql (e.g., immich_postgres.sql).
|
||||
// Small files — uses plain file copy, not rsync.
|
||||
// copyStackDBDumps copies DB dump files for the given stack from its home drive.
|
||||
// DB dumps are at <drive>/backups/primary/<stack>/db-dumps/<stack>_<dbtype>.sql.
|
||||
func (r *CrossDriveRunner) copyStackDBDumps(stackName, destDir string) error {
|
||||
if r.dbDumpDir == "" {
|
||||
return nil
|
||||
}
|
||||
entries, err := os.ReadDir(r.dbDumpDir)
|
||||
appDrive := r.getAppDrivePath(stackName)
|
||||
dumpDir := AppDBDumpPath(appDrive, stackName)
|
||||
|
||||
entries, err := os.ReadDir(dumpDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("reading DB dump dir: %w", err)
|
||||
}
|
||||
prefix := stackName + "_"
|
||||
|
||||
copied := 0
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasPrefix(e.Name(), prefix) {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
src := filepath.Join(r.dbDumpDir, e.Name())
|
||||
src := filepath.Join(dumpDir, e.Name())
|
||||
dst := filepath.Join(destDir, e.Name())
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
@@ -453,4 +456,3 @@ func dirSizeBytes(path string) (int64, error) {
|
||||
})
|
||||
return total, err
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package backup
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
// PrimaryBackupPath returns the root primary backup directory for a drive.
|
||||
func PrimaryBackupPath(drivePath string) string {
|
||||
return filepath.Join(drivePath, "backups", "primary")
|
||||
}
|
||||
|
||||
// PrimaryResticRepoPath returns the restic repo path on a drive's primary backup.
|
||||
func PrimaryResticRepoPath(drivePath string) string {
|
||||
return filepath.Join(drivePath, "backups", "primary", "restic")
|
||||
}
|
||||
|
||||
// AppDBDumpPath returns the DB dump directory for an app on its home drive.
|
||||
func AppDBDumpPath(drivePath, stackName string) string {
|
||||
return filepath.Join(drivePath, "backups", "primary", stackName, "db-dumps")
|
||||
}
|
||||
|
||||
// SecondaryBackupPath returns the root secondary backup directory for a drive.
|
||||
func SecondaryBackupPath(drivePath string) string {
|
||||
return filepath.Join(drivePath, "backups", "secondary")
|
||||
}
|
||||
|
||||
// AppSecondaryRsyncPath returns the rsync destination for an app's secondary backup.
|
||||
func AppSecondaryRsyncPath(drivePath, stackName string) string {
|
||||
return filepath.Join(drivePath, "backups", "secondary", stackName, "rsync")
|
||||
}
|
||||
|
||||
// SecondaryResticRepoPath returns the restic repo path on a drive's secondary backup.
|
||||
func SecondaryResticRepoPath(drivePath string) string {
|
||||
return filepath.Join(drivePath, "backups", "secondary", "restic")
|
||||
}
|
||||
|
||||
// AppDataDir returns the app data directory path on a drive.
|
||||
func AppDataDir(drivePath, stackName string) string {
|
||||
return filepath.Join(drivePath, "appdata", stackName)
|
||||
}
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
)
|
||||
|
||||
// ResticManager handles restic backup operations.
|
||||
// All methods accept repoPath as parameter to support per-drive repos.
|
||||
type ResticManager struct {
|
||||
repoPath string
|
||||
passwordFile string
|
||||
logger *log.Logger
|
||||
customerID string
|
||||
@@ -36,15 +36,17 @@ type SnapshotResult struct {
|
||||
|
||||
// SnapshotInfo holds information about a restic snapshot.
|
||||
type SnapshotInfo struct {
|
||||
ID string `json:"short_id"`
|
||||
Time time.Time `json:"time"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
ID string `json:"short_id"`
|
||||
Time time.Time `json:"time"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
RepoPath string `json:"-"` // set by caller for multi-repo aggregation
|
||||
}
|
||||
|
||||
// RepoStats holds repository statistics.
|
||||
type RepoStats struct {
|
||||
TotalSize string
|
||||
TotalSizeBytes int64
|
||||
SnapshotCount int
|
||||
LatestSnapshot *SnapshotInfo
|
||||
}
|
||||
@@ -52,7 +54,6 @@ type RepoStats struct {
|
||||
// NewResticManager creates a new restic manager.
|
||||
func NewResticManager(cfg *config.Config, logger *log.Logger) *ResticManager {
|
||||
return &ResticManager{
|
||||
repoPath: cfg.Backup.ResticRepo,
|
||||
passwordFile: cfg.Backup.ResticPasswordFile,
|
||||
logger: logger,
|
||||
customerID: cfg.Customer.ID,
|
||||
@@ -62,7 +63,7 @@ func NewResticManager(cfg *config.Config, logger *log.Logger) *ResticManager {
|
||||
|
||||
// EnsureInitialized checks if the restic repo exists and initializes it if not.
|
||||
// Also auto-generates the password file if missing.
|
||||
func (r *ResticManager) EnsureInitialized() error {
|
||||
func (r *ResticManager) EnsureInitialized(repoPath string) error {
|
||||
// Ensure password file exists
|
||||
if _, err := os.Stat(r.passwordFile); os.IsNotExist(err) {
|
||||
if err := r.generatePassword(); err != nil {
|
||||
@@ -74,23 +75,23 @@ func (r *ResticManager) EnsureInitialized() error {
|
||||
os.MkdirAll(r.cacheDir, 0700)
|
||||
|
||||
// Check if repo is already initialized
|
||||
configPath := filepath.Join(r.repoPath, "config")
|
||||
configPath := filepath.Join(repoPath, "config")
|
||||
if _, err := os.Stat(configPath); err == nil {
|
||||
r.logger.Printf("[INFO] Restic repo already initialized at %s", r.repoPath)
|
||||
r.logger.Printf("[INFO] Restic repo already initialized at %s", repoPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure repo directory exists
|
||||
if err := os.MkdirAll(r.repoPath, 0700); err != nil {
|
||||
if err := os.MkdirAll(repoPath, 0700); err != nil {
|
||||
return fmt.Errorf("creating repo dir: %w", err)
|
||||
}
|
||||
|
||||
// Initialize repo
|
||||
r.logger.Printf("[INFO] Initializing restic repository at %s", r.repoPath)
|
||||
r.logger.Printf("[INFO] Initializing restic repository at %s", repoPath)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := r.command(ctx, "init")
|
||||
cmd := r.command(ctx, repoPath, "init")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restic init failed: %v — %s", err, truncate(string(out), 200))
|
||||
@@ -101,7 +102,7 @@ func (r *ResticManager) EnsureInitialized() error {
|
||||
}
|
||||
|
||||
// Snapshot creates a new backup snapshot of the given paths.
|
||||
func (r *ResticManager) Snapshot(paths []string, tags []string) (*SnapshotResult, error) {
|
||||
func (r *ResticManager) Snapshot(repoPath string, paths []string, tags []string) (*SnapshotResult, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -128,17 +129,17 @@ func (r *ResticManager) Snapshot(paths []string, tags []string) (*SnapshotResult
|
||||
}
|
||||
args = append(args, existingPaths...)
|
||||
|
||||
cmd := r.command(ctx, args...)
|
||||
cmd := r.command(ctx, repoPath, args...)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
// Check for stale lock
|
||||
errStr := string(out)
|
||||
if strings.Contains(errStr, "lock") || strings.Contains(errStr, "locked") {
|
||||
r.logger.Printf("[WARN] Restic repo locked — attempting unlock")
|
||||
unlockCmd := r.command(ctx, "unlock")
|
||||
unlockCmd := r.command(ctx, repoPath, "unlock")
|
||||
unlockCmd.Run()
|
||||
// Retry once
|
||||
cmd = r.command(ctx, args...)
|
||||
cmd = r.command(ctx, repoPath, args...)
|
||||
out, err = cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("restic backup failed after unlock: %v", err)
|
||||
@@ -181,7 +182,7 @@ func (r *ResticManager) Snapshot(paths []string, tags []string) (*SnapshotResult
|
||||
}
|
||||
|
||||
// Prune removes old snapshots according to retention policy.
|
||||
func (r *ResticManager) Prune(retention config.RetentionConfig) error {
|
||||
func (r *ResticManager) Prune(repoPath string, retention config.RetentionConfig) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -193,22 +194,22 @@ func (r *ResticManager) Prune(retention config.RetentionConfig) error {
|
||||
"--prune",
|
||||
}
|
||||
|
||||
cmd := r.command(ctx, args...)
|
||||
cmd := r.command(ctx, repoPath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restic forget/prune failed: %v — %s", err, truncate(string(out), 200))
|
||||
}
|
||||
|
||||
r.logger.Printf("[INFO] Restic prune completed")
|
||||
r.logger.Printf("[INFO] Restic prune completed for %s", repoPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check verifies repository integrity.
|
||||
func (r *ResticManager) Check() error {
|
||||
func (r *ResticManager) Check(repoPath string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := r.command(ctx, "check")
|
||||
cmd := r.command(ctx, repoPath, "check")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restic check failed: %v — %s", err, truncate(string(out), 200))
|
||||
@@ -217,11 +218,11 @@ func (r *ResticManager) Check() error {
|
||||
}
|
||||
|
||||
// ListSnapshots returns all snapshots, newest first, limited to N entries.
|
||||
func (r *ResticManager) ListSnapshots(limit int) ([]SnapshotInfo, error) {
|
||||
func (r *ResticManager) ListSnapshots(repoPath string, limit int) ([]SnapshotInfo, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := r.command(ctx, "snapshots", "--json")
|
||||
cmd := r.command(ctx, repoPath, "snapshots", "--json")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("restic snapshots failed: %v", err)
|
||||
@@ -245,11 +246,11 @@ func (r *ResticManager) ListSnapshots(limit int) ([]SnapshotInfo, error) {
|
||||
}
|
||||
|
||||
// LatestSnapshot returns the most recent snapshot info.
|
||||
func (r *ResticManager) LatestSnapshot() (*SnapshotInfo, error) {
|
||||
func (r *ResticManager) LatestSnapshot(repoPath string) (*SnapshotInfo, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := r.command(ctx, "snapshots", "--latest", "1", "--json")
|
||||
cmd := r.command(ctx, repoPath, "snapshots", "--latest", "1", "--json")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("restic snapshots failed: %v", err)
|
||||
@@ -268,26 +269,27 @@ func (r *ResticManager) LatestSnapshot() (*SnapshotInfo, error) {
|
||||
}
|
||||
|
||||
// Stats returns repository statistics.
|
||||
func (r *ResticManager) Stats() (*RepoStats, error) {
|
||||
func (r *ResticManager) Stats(repoPath string) (*RepoStats, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
stats := &RepoStats{}
|
||||
|
||||
// Get repo size
|
||||
cmd := r.command(ctx, "stats", "--json")
|
||||
cmd := r.command(ctx, repoPath, "stats", "--json")
|
||||
out, err := cmd.Output()
|
||||
if err == nil {
|
||||
var raw struct {
|
||||
TotalSize uint64 `json:"total_size"`
|
||||
}
|
||||
if json.Unmarshal(out, &raw) == nil {
|
||||
stats.TotalSize = humanizeBytes(int64(raw.TotalSize))
|
||||
stats.TotalSizeBytes = int64(raw.TotalSize)
|
||||
stats.TotalSize = humanizeBytes(stats.TotalSizeBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Count snapshots
|
||||
cmd = r.command(ctx, "snapshots", "--json")
|
||||
cmd = r.command(ctx, repoPath, "snapshots", "--json")
|
||||
out, err = cmd.Output()
|
||||
if err == nil {
|
||||
var snapshots []SnapshotInfo
|
||||
@@ -313,7 +315,7 @@ func (r *ResticManager) GetPassword() (string, error) {
|
||||
}
|
||||
|
||||
// RestoreAppData restores specific paths from a restic snapshot.
|
||||
func (r *ResticManager) RestoreAppData(snapshotID string, paths []string) error {
|
||||
func (r *ResticManager) RestoreAppData(repoPath string, snapshotID string, paths []string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -325,9 +327,9 @@ func (r *ResticManager) RestoreAppData(snapshotID string, paths []string) error
|
||||
args = append(args, "--include", p)
|
||||
}
|
||||
|
||||
r.logger.Printf("[WARN] RESTORE started: snapshot=%s, paths=%v", snapshotID, paths)
|
||||
r.logger.Printf("[WARN] RESTORE started: repo=%s, snapshot=%s, paths=%v", repoPath, snapshotID, paths)
|
||||
|
||||
cmd := r.command(ctx, args...)
|
||||
cmd := r.command(ctx, repoPath, args...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
r.logger.Printf("[ERROR] Restore failed: %v, output: %s", err, truncate(string(output), 500))
|
||||
@@ -338,10 +340,16 @@ func (r *ResticManager) RestoreAppData(snapshotID string, paths []string) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ResticManager) command(ctx context.Context, args ...string) *exec.Cmd {
|
||||
// RepoExists checks if a restic repo is initialized at the given path.
|
||||
func (r *ResticManager) RepoExists(repoPath string) bool {
|
||||
_, err := os.Stat(filepath.Join(repoPath, "config"))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (r *ResticManager) command(ctx context.Context, repoPath string, args ...string) *exec.Cmd {
|
||||
cmd := exec.CommandContext(ctx, "restic", args...)
|
||||
cmd.Env = append(os.Environ(),
|
||||
"RESTIC_REPOSITORY="+r.repoPath,
|
||||
"RESTIC_REPOSITORY="+repoPath,
|
||||
"RESTIC_PASSWORD_FILE="+r.passwordFile,
|
||||
"RESTIC_CACHE_DIR="+r.cacheDir,
|
||||
)
|
||||
|
||||
@@ -49,10 +49,10 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
|
||||
restorePaths = append(restorePaths, stackDir)
|
||||
}
|
||||
|
||||
// Restore DB dump files for this stack
|
||||
if m.cfg.Paths.DBDumpDir != "" {
|
||||
restorePaths = append(restorePaths, m.cfg.Paths.DBDumpDir)
|
||||
}
|
||||
// Restore DB dump files for this stack (per-drive path)
|
||||
drivePath := m.GetAppDrivePath(stackName)
|
||||
dumpDir := AppDBDumpPath(drivePath, stackName)
|
||||
restorePaths = append(restorePaths, dumpDir)
|
||||
|
||||
// Restore HDD data (always included for apps that have it — backup is mandatory)
|
||||
if hasHDD {
|
||||
@@ -63,8 +63,11 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
|
||||
return fmt.Errorf("no restorable paths found for %s", stackName)
|
||||
}
|
||||
|
||||
m.logger.Printf("[WARN] RESTORE starting: stack=%s, snapshot=%s, paths=%v, hasHDD=%v",
|
||||
stackName, snapshotID, restorePaths, hasHDD)
|
||||
// Use the app's primary restic repo
|
||||
repoPath := PrimaryResticRepoPath(drivePath)
|
||||
|
||||
m.logger.Printf("[WARN] RESTORE starting: stack=%s, snapshot=%s, repo=%s, paths=%v, hasHDD=%v",
|
||||
stackName, snapshotID, repoPath, restorePaths, hasHDD)
|
||||
|
||||
// Stop the app before restore
|
||||
if err := m.stackProvider.StopStack(stackName); err != nil {
|
||||
@@ -72,7 +75,7 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
|
||||
}
|
||||
|
||||
// Execute restore via restic
|
||||
if err := m.restic.RestoreAppData(snapshotID, restorePaths); err != nil {
|
||||
if err := m.restic.RestoreAppData(repoPath, snapshotID, restorePaths); err != nil {
|
||||
m.logger.Printf("[ERROR] RESTORE failed for %s: %v", stackName, err)
|
||||
if startErr := m.stackProvider.StartStack(stackName); startErr != nil {
|
||||
m.logger.Printf("[WARN] RESTORE could not restart %s after failure: %v", stackName, startErr)
|
||||
|
||||
@@ -46,11 +46,10 @@ type InfrastructureConfig struct {
|
||||
}
|
||||
|
||||
type PathsConfig struct {
|
||||
StacksDir string `yaml:"stacks_dir"`
|
||||
DataDir string `yaml:"data_dir"`
|
||||
BackupDir string `yaml:"backup_dir"`
|
||||
DBDumpDir string `yaml:"db_dump_dir"`
|
||||
HDDPath string `yaml:"hdd_path"`
|
||||
StacksDir string `yaml:"stacks_dir"`
|
||||
DataDir string `yaml:"data_dir"`
|
||||
SystemDataPath string `yaml:"system_data_path"`
|
||||
HDDPath string `yaml:"hdd_path"`
|
||||
}
|
||||
|
||||
type WebConfig struct {
|
||||
@@ -75,7 +74,6 @@ type StacksConfig struct {
|
||||
|
||||
type BackupConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
ResticRepo string `yaml:"restic_repo"`
|
||||
ResticPasswordFile string `yaml:"restic_password_file"`
|
||||
DBDumpSchedule string `yaml:"db_dump_schedule"`
|
||||
ResticSchedule string `yaml:"restic_schedule"`
|
||||
@@ -185,13 +183,11 @@ func applyDefaults(cfg *Config) {
|
||||
|
||||
d(&cfg.Paths.StacksDir, "/opt/docker/stacks")
|
||||
d(&cfg.Paths.DataDir, "/opt/docker/felhom-controller/data")
|
||||
d(&cfg.Paths.BackupDir, "/srv/backups")
|
||||
d(&cfg.Paths.DBDumpDir, "/srv/backups/db-dumps")
|
||||
d(&cfg.Paths.SystemDataPath, "/mnt/sys_drive")
|
||||
d(&cfg.Web.Listen, ":8080")
|
||||
d(&cfg.Git.Branch, "main")
|
||||
d(&cfg.Git.SyncInterval, "15m")
|
||||
d(&cfg.Stacks.UpdateWindow, "03:00-05:00")
|
||||
d(&cfg.Backup.ResticRepo, "/srv/backups/restic-repo")
|
||||
d(&cfg.Backup.DBDumpSchedule, "02:30")
|
||||
d(&cfg.Backup.ResticSchedule, "03:00")
|
||||
d(&cfg.Backup.PruneSchedule, "weekly")
|
||||
|
||||
@@ -41,10 +41,10 @@ func ProtectedHDDPaths(hddPath string) map[string]bool {
|
||||
}
|
||||
return map[string]bool{
|
||||
hddPath: true,
|
||||
filepath.Join(hddPath, "media"): true,
|
||||
filepath.Join(hddPath, "storage"): true,
|
||||
filepath.Join(hddPath, "Dokumentumok"): true,
|
||||
filepath.Join(hddPath, "appdata"): true,
|
||||
filepath.Join(hddPath, "backups"): true,
|
||||
filepath.Join(hddPath, "media"): true,
|
||||
filepath.Join(hddPath, "Dokumentumok"): true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -525,8 +525,7 @@ func (s *Server) backupsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
data["ResticPassword"] = pw
|
||||
}
|
||||
|
||||
// Tároló section: DB dump directory and total size
|
||||
data["DBDumpDir"] = s.cfg.Paths.DBDumpDir
|
||||
// Tároló section: DB dump total size
|
||||
var dbDumpTotalBytes int64
|
||||
for _, f := range fullStatus.DumpFiles {
|
||||
dbDumpTotalBytes += f.Size
|
||||
|
||||
@@ -377,14 +377,10 @@
|
||||
<div class="repo-card">
|
||||
<h3>Tároló</h3>
|
||||
|
||||
<!-- Tier 1: Local restic backup -->
|
||||
<!-- Tier 1: Local restic backup (per-drive) -->
|
||||
<div class="repo-tier">
|
||||
<h4 class="repo-tier-title">1. mentés — Helyi mentés (restic)</h4>
|
||||
<div class="repo-info-rows">
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Helyszín:</span>
|
||||
<span class="repo-value mono">{{.Backup.RepoPath}} <span class="relative-time">(helyi SSD)</span></span>
|
||||
</div>
|
||||
{{if .Backup.RepoStats}}
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Méret:</span>
|
||||
@@ -395,6 +391,10 @@
|
||||
<span class="repo-value">{{.Backup.RepoStats.SnapshotCount}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Adatbázis mentések:</span>
|
||||
<span class="repo-value">{{if .Backup.DumpFiles}}{{len .Backup.DumpFiles}} dump fájl{{if gt .DBDumpTotalBytes 0}} — {{fmtBytes .DBDumpTotalBytes}}{{end}}{{else}}Nincs dump fájl{{end}}</span>
|
||||
</div>
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Integritás:</span>
|
||||
<span class="repo-value">
|
||||
@@ -423,15 +423,6 @@
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<div class="repo-paths">
|
||||
<span class="repo-label">Mentett útvonalak (forrás):</span>
|
||||
<ul class="repo-path-list">
|
||||
{{range .Backup.BackupPaths}}
|
||||
<li class="mono">{{.}}</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Tier 2: Cross-drive backup destinations -->
|
||||
@@ -458,21 +449,6 @@
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<!-- DB dump storage -->
|
||||
<div class="repo-tier">
|
||||
<h4 class="repo-tier-title">Adatbázis mentések</h4>
|
||||
<div class="repo-info-rows">
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Mappa:</span>
|
||||
<span class="repo-value mono">{{.DBDumpDir}}</span>
|
||||
</div>
|
||||
<div class="repo-info-row">
|
||||
<span class="repo-label">Fájlok:</span>
|
||||
<span class="repo-value">{{if .Backup.DumpFiles}}{{len .Backup.DumpFiles}} dump fájl{{if gt .DBDumpTotalBytes 0}} — {{fmtBytes .DBDumpTotalBytes}}{{end}}{{else}}Nincs dump fájl{{end}}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Section 7: Restore -->
|
||||
|
||||
Reference in New Issue
Block a user