v0.15.5: Disaster recovery — Hub-based infra backup, auto-mount, restore UI
Complete DR implementation (TASK2.md Phases 1-4): - Hub infra-backup push/pull endpoints (controller.yaml, disk layout, stacks) - Fresh-deployment detection pulls config from Hub, auto-mounts drives by UUID - Full-page restore UI with drive status, app table, sequential restore - docker-setup.sh shows DR instructions when customer_id is configured New files: disk_layout.go, restore_scan.go, restore_app_linux.go, restore_drives_linux.go, infra_backup.go, infra_pull.go, handler_restore.go, restore.html Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
package backup
|
||||
|
||||
// DiskLayout holds the fstab-derived mount topology for disaster recovery.
|
||||
type DiskLayout struct {
|
||||
Mounts []DiskMount `json:"mounts"`
|
||||
}
|
||||
|
||||
// DiskMount represents a single mount entry from fstab.
|
||||
type DiskMount struct {
|
||||
UUID string `json:"uuid"`
|
||||
Label string `json:"label"`
|
||||
MountPoint string `json:"mount_point"`
|
||||
FSType string `json:"fs_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
FstabOptions string `json:"fstab_options"`
|
||||
Role string `json:"role"` // "system_data", "hdd_storage"
|
||||
BindSubdir string `json:"bind_subdir"` // e.g., "felhom_data"
|
||||
RawMount string `json:"raw_mount"` // e.g., "/mnt/.felhom-raw/hdd_1"
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
//go:build linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RestoreAppFromBackup restores a single app from its cross-drive backup.
|
||||
// Steps: restore config → verify/restore data → copy DB dumps → docker compose up.
|
||||
func RestoreAppFromBackup(ctx context.Context, app *RestorableApp, stacksDir string, logger *log.Logger) error {
|
||||
stackDir := filepath.Join(stacksDir, app.Name)
|
||||
|
||||
// Step 1: Restore stack config from _config/ backup
|
||||
if app.HasConfig {
|
||||
logger.Printf("[INFO] Restoring config for %s from %s", app.Name, app.ConfigPath)
|
||||
if err := restoreConfigDir(ctx, app.ConfigPath, stackDir); err != nil {
|
||||
return fmt.Errorf("restoring config: %w", err)
|
||||
}
|
||||
} else {
|
||||
// No config backup — check if stack dir already exists (from catalog sync)
|
||||
if !dirExists(stackDir) {
|
||||
return fmt.Errorf("no config backup and no stack directory for %s", app.Name)
|
||||
}
|
||||
logger.Printf("[INFO] No config backup for %s — using existing stack dir", app.Name)
|
||||
}
|
||||
|
||||
// Step 2: Verify app data on HDD (common case: HDD survived, data is intact)
|
||||
if app.NeedsHDD && !app.HasData && app.HasRsyncData {
|
||||
// App data is missing but rsync backup exists — restore it
|
||||
logger.Printf("[INFO] Restoring user data for %s from rsync backup", app.Name)
|
||||
if err := restoreUserData(ctx, app, logger); err != nil {
|
||||
logger.Printf("[WARN] User data restore failed for %s: %v", app.Name, err)
|
||||
// Non-fatal: app might still start without all data
|
||||
}
|
||||
} else if app.HasData {
|
||||
logger.Printf("[INFO] App data for %s found at %s — no restore needed", app.Name, app.DataPath)
|
||||
}
|
||||
|
||||
// Step 3: Copy DB dumps to primary backup location
|
||||
if app.HasDBDump {
|
||||
logger.Printf("[INFO] Restoring DB dumps for %s", app.Name)
|
||||
if err := restoreDBDumps(app, logger); err != nil {
|
||||
logger.Printf("[WARN] DB dump restore failed for %s: %v", app.Name, err)
|
||||
// Non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Docker compose pull + up
|
||||
composePath := filepath.Join(stackDir, "docker-compose.yml")
|
||||
if !fileExistsCheck(composePath) {
|
||||
composePath = filepath.Join(stackDir, "compose.yml")
|
||||
if !fileExistsCheck(composePath) {
|
||||
return fmt.Errorf("no compose file found in %s", stackDir)
|
||||
}
|
||||
}
|
||||
|
||||
composeDir := filepath.Dir(composePath)
|
||||
|
||||
logger.Printf("[INFO] Pulling images for %s", app.Name)
|
||||
pullCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "pull")
|
||||
pullCmd.Dir = composeDir
|
||||
if out, err := pullCmd.CombinedOutput(); err != nil {
|
||||
logger.Printf("[WARN] docker compose pull failed for %s: %v (%s)", app.Name, err, strings.TrimSpace(string(out)))
|
||||
// Non-fatal: might work with cached images
|
||||
}
|
||||
|
||||
logger.Printf("[INFO] Starting %s", app.Name)
|
||||
upCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "up", "-d")
|
||||
upCmd.Dir = composeDir
|
||||
if out, err := upCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("docker compose up: %v (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreConfigDir rsyncs the backed-up _config/ directory to the stack directory.
|
||||
func restoreConfigDir(ctx context.Context, configBackupDir, stackDir string) error {
|
||||
if err := os.MkdirAll(stackDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating stack dir: %w", err)
|
||||
}
|
||||
|
||||
src := strings.TrimRight(configBackupDir, "/") + "/"
|
||||
dst := strings.TrimRight(stackDir, "/") + "/"
|
||||
|
||||
cmd := exec.CommandContext(ctx, "rsync", "-a", src, dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("rsync config: %v (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreUserData rsyncs user data from cross-drive backup back to the app's HDD path.
|
||||
func restoreUserData(ctx context.Context, app *RestorableApp, logger *log.Logger) error {
|
||||
if app.RsyncDataPath == "" || app.HDDPath == "" {
|
||||
return fmt.Errorf("no rsync data path or HDD path")
|
||||
}
|
||||
|
||||
// The rsync backup contains the app's data directories.
|
||||
// Walk the backup dir and rsync each subdirectory (excluding _config/_db)
|
||||
// back to the app's HDD data directory.
|
||||
entries, err := os.ReadDir(app.RsyncDataPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataDir := AppDataDir(app.HDDPath, app.Name)
|
||||
if err := os.MkdirAll(dataDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating data dir: %w", err)
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if name == "_config" || name == "_db" || strings.HasPrefix(name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
src := filepath.Join(app.RsyncDataPath, name)
|
||||
dst := filepath.Join(dataDir, name)
|
||||
|
||||
if e.IsDir() {
|
||||
src = strings.TrimRight(src, "/") + "/"
|
||||
if err := os.MkdirAll(dst, 0755); err != nil {
|
||||
logger.Printf("[WARN] Cannot create %s: %v", dst, err)
|
||||
continue
|
||||
}
|
||||
dst = strings.TrimRight(dst, "/") + "/"
|
||||
cmd := exec.CommandContext(ctx, "rsync", "-a", src, dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
logger.Printf("[WARN] rsync data %s: %v (%s)", name, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
} else {
|
||||
// Single file — copy directly
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] Cannot read %s: %v", src, err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
||||
logger.Printf("[WARN] Cannot write %s: %v", dst, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreDBDumps copies DB dump files from cross-drive backup to the primary dump dir.
|
||||
func restoreDBDumps(app *RestorableApp, logger *log.Logger) error {
|
||||
if app.DBDumpPath == "" || app.HDDPath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
destDir := AppDBDumpPath(app.HDDPath, app.Name)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating dump dir: %w", err)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(app.DBDumpPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
src := filepath.Join(app.DBDumpPath, e.Name())
|
||||
dst := filepath.Join(destDir, e.Name())
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] Cannot read dump %s: %v", e.Name(), err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
||||
logger.Printf("[WARN] Cannot write dump %s: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fileExistsCheck returns true if path exists and is a file.
|
||||
func fileExistsCheck(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//go:build !linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
)
|
||||
|
||||
// RestoreAppFromBackup is a no-op on non-Linux platforms.
|
||||
func RestoreAppFromBackup(ctx context.Context, app *RestorableApp, stacksDir string, logger *log.Logger) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//go:build linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MountDrivesFromLayout scans block devices for disks matching the stored
|
||||
// disk layout and mounts them using the felhom two-layer mount pattern
|
||||
// (raw mount → bind mount).
|
||||
//
|
||||
// The controller container runs privileged with:
|
||||
// - /host-dev mounted from host /dev
|
||||
// - /host-fstab mounted from host /etc/fstab
|
||||
// - /mnt with rshared propagation
|
||||
//
|
||||
// Returns the list of successfully mounted final mount paths.
|
||||
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
|
||||
if len(layout.Mounts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Get current block devices with UUIDs
|
||||
uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning block devices: %w", err)
|
||||
}
|
||||
|
||||
var mounted []string
|
||||
|
||||
for _, dm := range layout.Mounts {
|
||||
if dm.UUID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Find matching device by UUID
|
||||
device := uuidToDevice[dm.UUID]
|
||||
if device == "" {
|
||||
logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
|
||||
dm.UUID, dm.Label)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if already mounted
|
||||
finalMount := dm.MountPoint
|
||||
if isMountedPath(finalMount) {
|
||||
logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
|
||||
mounted = append(mounted, finalMount)
|
||||
continue
|
||||
}
|
||||
if dm.RawMount != "" && isMountedPath(dm.RawMount) {
|
||||
logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
|
||||
mounted = append(mounted, finalMount)
|
||||
continue
|
||||
}
|
||||
|
||||
uuidShort := dm.UUID
|
||||
if len(uuidShort) > 12 {
|
||||
uuidShort = uuidShort[:12]
|
||||
}
|
||||
logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
|
||||
device, uuidShort, dm.Label, finalMount)
|
||||
|
||||
// Mount using the appropriate pattern
|
||||
if dm.RawMount != "" && dm.BindSubdir != "" {
|
||||
// Two-layer HDD mount: raw → bind
|
||||
if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
|
||||
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
// Simple direct mount (e.g., sys_drive)
|
||||
if err := mountDirect(ctx, device, dm, logger); err != nil {
|
||||
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Update host fstab so mount persists across reboots
|
||||
if err := addDRFstabEntries(dm, logger); err != nil {
|
||||
logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
|
||||
}
|
||||
|
||||
mounted = append(mounted, finalMount)
|
||||
logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
|
||||
}
|
||||
|
||||
return mounted, nil
|
||||
}
|
||||
|
||||
// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID → device path map.
|
||||
func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
|
||||
// First try lsblk with UUID output
|
||||
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("lsblk failed: %w", err)
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
BlockDevices []struct {
|
||||
Name string `json:"name"`
|
||||
UUID *string `json:"uuid"`
|
||||
FSType *string `json:"fstype"`
|
||||
Mount *string `json:"mountpoint"`
|
||||
Children []struct {
|
||||
Name string `json:"name"`
|
||||
UUID *string `json:"uuid"`
|
||||
FSType *string `json:"fstype"`
|
||||
Mount *string `json:"mountpoint"`
|
||||
} `json:"children"`
|
||||
} `json:"blockdevices"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &parsed); err != nil {
|
||||
return nil, fmt.Errorf("lsblk parse failed: %w", err)
|
||||
}
|
||||
|
||||
devices := make(map[string]string) // UUID → /dev/path
|
||||
for _, dev := range parsed.BlockDevices {
|
||||
if dev.UUID != nil && *dev.UUID != "" {
|
||||
devices[*dev.UUID] = "/dev/" + dev.Name
|
||||
}
|
||||
for _, child := range dev.Children {
|
||||
if child.UUID != nil && *child.UUID != "" {
|
||||
devices[*child.UUID] = "/dev/" + child.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
|
||||
if len(devices) == 0 {
|
||||
// Try blkid on /host-dev devices
|
||||
blkOut, err := exec.CommandContext(ctx, "blkid").Output()
|
||||
if err == nil {
|
||||
for _, line := range strings.Split(string(blkOut), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
|
||||
colonIdx := strings.Index(line, ":")
|
||||
if colonIdx < 0 {
|
||||
continue
|
||||
}
|
||||
devPath := line[:colonIdx]
|
||||
if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
|
||||
rest := line[uuidIdx+6:]
|
||||
if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
|
||||
uuid := rest[:endIdx]
|
||||
devices[uuid] = devPath
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
// mountDirect creates a simple direct mount.
|
||||
func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
|
||||
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
|
||||
return fmt.Errorf("creating mount point: %w", err)
|
||||
}
|
||||
|
||||
// Use host device path if available
|
||||
devPath := hostDevPath(device)
|
||||
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mountRawAndBind implements the two-layer felhom mount pattern.
|
||||
func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
|
||||
// Layer 1: raw mount
|
||||
if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
|
||||
return fmt.Errorf("creating raw mount point: %w", err)
|
||||
}
|
||||
|
||||
devPath := hostDevPath(device)
|
||||
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("raw mount %s → %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
|
||||
// Layer 2: bind mount (subdir → final mount point)
|
||||
bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
|
||||
if err := os.MkdirAll(bindSrc, 0755); err != nil {
|
||||
return fmt.Errorf("creating bind source dir: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
|
||||
return fmt.Errorf("creating final mount point: %w", err)
|
||||
}
|
||||
|
||||
cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("bind mount %s → %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
|
||||
func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
|
||||
const fstabPath = "/host-fstab"
|
||||
|
||||
data, err := os.ReadFile(fstabPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading fstab: %w", err)
|
||||
}
|
||||
|
||||
content := string(data)
|
||||
|
||||
// Skip if UUID already in fstab (idempotent)
|
||||
if strings.Contains(content, dm.UUID) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var additions strings.Builder
|
||||
additions.WriteString("\n# Restored by felhom-controller DR\n")
|
||||
|
||||
if dm.RawMount != "" {
|
||||
// Raw mount entry
|
||||
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
|
||||
dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
|
||||
}
|
||||
|
||||
if dm.BindSubdir != "" && dm.RawMount != "" {
|
||||
// Bind mount entry
|
||||
additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
|
||||
dm.RawMount, dm.BindSubdir, dm.MountPoint))
|
||||
} else if dm.RawMount == "" {
|
||||
// Direct mount entry (no bind)
|
||||
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
|
||||
dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
|
||||
}
|
||||
|
||||
newContent := content + additions.String()
|
||||
|
||||
// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
|
||||
tmpPath := fstabPath + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing fstab tmp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, fstabPath); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
// Fallback: direct write (bind-mounted files can't be renamed)
|
||||
if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing fstab: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isMountedPath checks if a path is currently a mount point via /proc/mounts.
|
||||
func isMountedPath(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
data, err := os.ReadFile("/proc/mounts")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
cleanPath := filepath.Clean(path)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
|
||||
func hostDevPath(devPath string) string {
|
||||
if strings.HasPrefix(devPath, "/dev/") {
|
||||
return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
|
||||
}
|
||||
return devPath
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//go:build !linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
)
|
||||
|
||||
// MountDrivesFromLayout is a no-op on non-Linux platforms.
|
||||
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RestorableApp describes an app that can be restored during DR.
|
||||
type RestorableApp struct {
|
||||
Name string `json:"name"`
|
||||
DisplayName string `json:"display_name"`
|
||||
NeedsHDD bool `json:"needs_hdd"`
|
||||
HDDPath string `json:"hdd_path,omitempty"`
|
||||
|
||||
// What was found on disk
|
||||
HasConfig bool `json:"has_config"` // _config/ dir with compose files
|
||||
ConfigPath string `json:"config_path"` // full path to _config/ backup
|
||||
HasData bool `json:"has_data"` // app data dir exists on HDD
|
||||
DataPath string `json:"data_path"` // e.g., /mnt/hdd_1/appdata/immich
|
||||
HasDBDump bool `json:"has_db_dump"` // _db/ dir with dump files
|
||||
DBDumpPath string `json:"db_dump_path"` // full path to _db/ backup
|
||||
HasRsyncData bool `json:"has_rsync_data"` // rsync user data (excl _config/_db)
|
||||
RsyncDataPath string `json:"rsync_data_path"` // full path to rsync backup
|
||||
DrivePath string `json:"drive_path"` // which drive has the backup
|
||||
DriveLabel string `json:"drive_label"` // label for display
|
||||
|
||||
// Restore progress (updated during restore)
|
||||
Status string `json:"status"` // "pending", "restoring", "done", "failed", "skipped"
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt string `json:"started_at,omitempty"`
|
||||
CompletedAt string `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
// RestorePlan holds the complete DR restore plan.
|
||||
type RestorePlan struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
CustomerID string `json:"customer_id"`
|
||||
Domain string `json:"domain"`
|
||||
Timestamp string `json:"timestamp"` // when the infra backup was taken
|
||||
Apps []RestorableApp `json:"apps"`
|
||||
|
||||
// Drive summary
|
||||
Drives []DriveInfo `json:"drives"`
|
||||
|
||||
// Overall status
|
||||
Status string `json:"status"` // "pending", "restoring", "done"
|
||||
}
|
||||
|
||||
// DriveInfo summarizes a mounted drive for display.
|
||||
type DriveInfo struct {
|
||||
Path string `json:"path"`
|
||||
Label string `json:"label"`
|
||||
Available bool `json:"available"` // mount is accessible
|
||||
HasBackup bool `json:"has_backup"` // has backups/secondary/ dir
|
||||
}
|
||||
|
||||
// GetApps returns a snapshot of the apps list.
|
||||
func (rp *RestorePlan) GetApps() []RestorableApp {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
apps := make([]RestorableApp, len(rp.Apps))
|
||||
copy(apps, rp.Apps)
|
||||
return apps
|
||||
}
|
||||
|
||||
// Snapshot returns a thread-safe snapshot of the plan for JSON serialization.
|
||||
func (rp *RestorePlan) Snapshot() map[string]interface{} {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
return map[string]interface{}{
|
||||
"ok": true,
|
||||
"status": rp.Status,
|
||||
"apps": rp.Apps,
|
||||
"drives": rp.Drives,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateApp updates a single app's status in the plan.
|
||||
func (rp *RestorePlan) UpdateApp(name, status, errMsg string) {
|
||||
rp.mu.Lock()
|
||||
defer rp.mu.Unlock()
|
||||
for i := range rp.Apps {
|
||||
if rp.Apps[i].Name == name {
|
||||
rp.Apps[i].Status = status
|
||||
rp.Apps[i].Error = errMsg
|
||||
if status == "restoring" {
|
||||
rp.Apps[i].StartedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
if status == "done" || status == "failed" {
|
||||
rp.Apps[i].CompletedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AllDone returns true if all apps are done/failed/skipped.
|
||||
func (rp *RestorePlan) AllDone() bool {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
for _, app := range rp.Apps {
|
||||
if app.Status != "done" && app.Status != "failed" && app.Status != "skipped" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// InfraStackInfo is a minimal stack descriptor from the Hub infra backup.
|
||||
// Used to pass deployed_stacks info into the scan without importing report.
|
||||
type InfraStackInfo struct {
|
||||
Name string
|
||||
DisplayName string
|
||||
HDDPath string
|
||||
NeedsHDD bool
|
||||
}
|
||||
|
||||
// ScanDrivesForBackups scans mounted drives for cross-drive backup data
|
||||
// and correlates with the deployed stacks manifest from the Hub.
|
||||
func ScanDrivesForBackups(mountedPaths []string, stacks []InfraStackInfo, logger *log.Logger) *RestorePlan {
|
||||
plan := &RestorePlan{
|
||||
Status: "pending",
|
||||
}
|
||||
|
||||
// Build drive info and find backup directories
|
||||
type driveBackup struct {
|
||||
drivePath string
|
||||
label string
|
||||
secPath string // backups/secondary/ path
|
||||
}
|
||||
var backupDrives []driveBackup
|
||||
|
||||
for _, mp := range mountedPaths {
|
||||
label := filepath.Base(mp)
|
||||
avail := dirExists(mp)
|
||||
|
||||
di := DriveInfo{
|
||||
Path: mp,
|
||||
Label: label,
|
||||
Available: avail,
|
||||
}
|
||||
|
||||
secPath := SecondaryBackupPath(mp)
|
||||
if dirExists(secPath) {
|
||||
di.HasBackup = true
|
||||
backupDrives = append(backupDrives, driveBackup{
|
||||
drivePath: mp,
|
||||
label: label,
|
||||
secPath: secPath,
|
||||
})
|
||||
logger.Printf("[INFO] Found backup data on %s (%s)", mp, secPath)
|
||||
}
|
||||
|
||||
plan.Drives = append(plan.Drives, di)
|
||||
}
|
||||
|
||||
// For each stack from the manifest, look for backup data on drives
|
||||
for _, stack := range stacks {
|
||||
app := RestorableApp{
|
||||
Name: stack.Name,
|
||||
DisplayName: stack.DisplayName,
|
||||
NeedsHDD: stack.NeedsHDD,
|
||||
HDDPath: stack.HDDPath,
|
||||
Status: "pending",
|
||||
}
|
||||
|
||||
// Check if app data exists directly on HDD (common case: HDD survived)
|
||||
if stack.HDDPath != "" {
|
||||
dataDir := AppDataDir(stack.HDDPath, stack.Name)
|
||||
if dirExists(dataDir) {
|
||||
app.HasData = true
|
||||
app.DataPath = dataDir
|
||||
}
|
||||
}
|
||||
|
||||
// Scan each drive for cross-drive backup of this app
|
||||
for _, db := range backupDrives {
|
||||
rsyncBase := AppSecondaryRsyncPath(db.drivePath, stack.Name)
|
||||
if !dirExists(rsyncBase) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Found a backup for this app
|
||||
app.DrivePath = db.drivePath
|
||||
app.DriveLabel = db.label
|
||||
|
||||
// Check for _config/ (stack compose directory backup)
|
||||
configDir := filepath.Join(rsyncBase, "_config")
|
||||
if dirExists(configDir) {
|
||||
app.HasConfig = true
|
||||
app.ConfigPath = configDir
|
||||
}
|
||||
|
||||
// Check for _db/ (database dump backup)
|
||||
dbDir := filepath.Join(rsyncBase, "_db")
|
||||
if dirExists(dbDir) && !dirIsEmpty(dbDir) {
|
||||
app.HasDBDump = true
|
||||
app.DBDumpPath = dbDir
|
||||
}
|
||||
|
||||
// Check for user data in rsync (anything besides _config and _db)
|
||||
if hasUserData(rsyncBase) {
|
||||
app.HasRsyncData = true
|
||||
app.RsyncDataPath = rsyncBase
|
||||
}
|
||||
|
||||
break // use first drive with backup for this app
|
||||
}
|
||||
|
||||
plan.Apps = append(plan.Apps, app)
|
||||
}
|
||||
|
||||
if len(plan.Apps) == 0 {
|
||||
plan.Apps = []RestorableApp{}
|
||||
}
|
||||
|
||||
logger.Printf("[INFO] Restore plan: %d apps, %d drives (%d with backups)",
|
||||
len(plan.Apps), len(plan.Drives), len(backupDrives))
|
||||
|
||||
return plan
|
||||
}
|
||||
|
||||
// dirExists checks if a directory exists and is accessible.
|
||||
func dirExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
|
||||
// dirIsEmpty returns true if a directory has no entries.
|
||||
func dirIsEmpty(path string) bool {
|
||||
entries, err := os.ReadDir(path)
|
||||
return err != nil || len(entries) == 0
|
||||
}
|
||||
|
||||
// hasUserData checks if the rsync backup dir has user data (not just _config/_db).
|
||||
func hasUserData(rsyncBase string) bool {
|
||||
entries, err := os.ReadDir(rsyncBase)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if name != "_config" && name != "_db" && !hasPrefix(name, ".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasPrefix(s, prefix string) bool {
|
||||
return len(s) >= len(prefix) && s[:len(prefix)] == prefix
|
||||
}
|
||||
Reference in New Issue
Block a user