v0.15.5: Disaster recovery — Hub-based infra backup, auto-mount, restore UI
Complete DR implementation (TASK2.md Phases 1-4): - Hub infra-backup push/pull endpoints (controller.yaml, disk layout, stacks) - Fresh-deployment detection pulls config from Hub, auto-mounts drives by UUID - Full-page restore UI with drive status, app table, sequential restore - docker-setup.sh shows DR instructions when customer_id is configured New files: disk_layout.go, restore_scan.go, restore_app_linux.go, restore_drives_linux.go, infra_backup.go, infra_pull.go, handler_restore.go, restore.html Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RestorableApp describes an app that can be restored during DR.
|
||||
type RestorableApp struct {
|
||||
Name string `json:"name"`
|
||||
DisplayName string `json:"display_name"`
|
||||
NeedsHDD bool `json:"needs_hdd"`
|
||||
HDDPath string `json:"hdd_path,omitempty"`
|
||||
|
||||
// What was found on disk
|
||||
HasConfig bool `json:"has_config"` // _config/ dir with compose files
|
||||
ConfigPath string `json:"config_path"` // full path to _config/ backup
|
||||
HasData bool `json:"has_data"` // app data dir exists on HDD
|
||||
DataPath string `json:"data_path"` // e.g., /mnt/hdd_1/appdata/immich
|
||||
HasDBDump bool `json:"has_db_dump"` // _db/ dir with dump files
|
||||
DBDumpPath string `json:"db_dump_path"` // full path to _db/ backup
|
||||
HasRsyncData bool `json:"has_rsync_data"` // rsync user data (excl _config/_db)
|
||||
RsyncDataPath string `json:"rsync_data_path"` // full path to rsync backup
|
||||
DrivePath string `json:"drive_path"` // which drive has the backup
|
||||
DriveLabel string `json:"drive_label"` // label for display
|
||||
|
||||
// Restore progress (updated during restore)
|
||||
Status string `json:"status"` // "pending", "restoring", "done", "failed", "skipped"
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt string `json:"started_at,omitempty"`
|
||||
CompletedAt string `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
// RestorePlan holds the complete DR restore plan.
|
||||
type RestorePlan struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
CustomerID string `json:"customer_id"`
|
||||
Domain string `json:"domain"`
|
||||
Timestamp string `json:"timestamp"` // when the infra backup was taken
|
||||
Apps []RestorableApp `json:"apps"`
|
||||
|
||||
// Drive summary
|
||||
Drives []DriveInfo `json:"drives"`
|
||||
|
||||
// Overall status
|
||||
Status string `json:"status"` // "pending", "restoring", "done"
|
||||
}
|
||||
|
||||
// DriveInfo summarizes a mounted drive for display.
|
||||
type DriveInfo struct {
|
||||
Path string `json:"path"`
|
||||
Label string `json:"label"`
|
||||
Available bool `json:"available"` // mount is accessible
|
||||
HasBackup bool `json:"has_backup"` // has backups/secondary/ dir
|
||||
}
|
||||
|
||||
// GetApps returns a snapshot of the apps list.
|
||||
func (rp *RestorePlan) GetApps() []RestorableApp {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
apps := make([]RestorableApp, len(rp.Apps))
|
||||
copy(apps, rp.Apps)
|
||||
return apps
|
||||
}
|
||||
|
||||
// Snapshot returns a thread-safe snapshot of the plan for JSON serialization.
|
||||
func (rp *RestorePlan) Snapshot() map[string]interface{} {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
return map[string]interface{}{
|
||||
"ok": true,
|
||||
"status": rp.Status,
|
||||
"apps": rp.Apps,
|
||||
"drives": rp.Drives,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateApp updates a single app's status in the plan.
|
||||
func (rp *RestorePlan) UpdateApp(name, status, errMsg string) {
|
||||
rp.mu.Lock()
|
||||
defer rp.mu.Unlock()
|
||||
for i := range rp.Apps {
|
||||
if rp.Apps[i].Name == name {
|
||||
rp.Apps[i].Status = status
|
||||
rp.Apps[i].Error = errMsg
|
||||
if status == "restoring" {
|
||||
rp.Apps[i].StartedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
if status == "done" || status == "failed" {
|
||||
rp.Apps[i].CompletedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AllDone returns true if all apps are done/failed/skipped.
|
||||
func (rp *RestorePlan) AllDone() bool {
|
||||
rp.mu.RLock()
|
||||
defer rp.mu.RUnlock()
|
||||
for _, app := range rp.Apps {
|
||||
if app.Status != "done" && app.Status != "failed" && app.Status != "skipped" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// InfraStackInfo is a minimal stack descriptor from the Hub infra backup.
|
||||
// Used to pass deployed_stacks info into the scan without importing report.
|
||||
type InfraStackInfo struct {
|
||||
Name string
|
||||
DisplayName string
|
||||
HDDPath string
|
||||
NeedsHDD bool
|
||||
}
|
||||
|
||||
// ScanDrivesForBackups scans mounted drives for cross-drive backup data
|
||||
// and correlates with the deployed stacks manifest from the Hub.
|
||||
func ScanDrivesForBackups(mountedPaths []string, stacks []InfraStackInfo, logger *log.Logger) *RestorePlan {
|
||||
plan := &RestorePlan{
|
||||
Status: "pending",
|
||||
}
|
||||
|
||||
// Build drive info and find backup directories
|
||||
type driveBackup struct {
|
||||
drivePath string
|
||||
label string
|
||||
secPath string // backups/secondary/ path
|
||||
}
|
||||
var backupDrives []driveBackup
|
||||
|
||||
for _, mp := range mountedPaths {
|
||||
label := filepath.Base(mp)
|
||||
avail := dirExists(mp)
|
||||
|
||||
di := DriveInfo{
|
||||
Path: mp,
|
||||
Label: label,
|
||||
Available: avail,
|
||||
}
|
||||
|
||||
secPath := SecondaryBackupPath(mp)
|
||||
if dirExists(secPath) {
|
||||
di.HasBackup = true
|
||||
backupDrives = append(backupDrives, driveBackup{
|
||||
drivePath: mp,
|
||||
label: label,
|
||||
secPath: secPath,
|
||||
})
|
||||
logger.Printf("[INFO] Found backup data on %s (%s)", mp, secPath)
|
||||
}
|
||||
|
||||
plan.Drives = append(plan.Drives, di)
|
||||
}
|
||||
|
||||
// For each stack from the manifest, look for backup data on drives
|
||||
for _, stack := range stacks {
|
||||
app := RestorableApp{
|
||||
Name: stack.Name,
|
||||
DisplayName: stack.DisplayName,
|
||||
NeedsHDD: stack.NeedsHDD,
|
||||
HDDPath: stack.HDDPath,
|
||||
Status: "pending",
|
||||
}
|
||||
|
||||
// Check if app data exists directly on HDD (common case: HDD survived)
|
||||
if stack.HDDPath != "" {
|
||||
dataDir := AppDataDir(stack.HDDPath, stack.Name)
|
||||
if dirExists(dataDir) {
|
||||
app.HasData = true
|
||||
app.DataPath = dataDir
|
||||
}
|
||||
}
|
||||
|
||||
// Scan each drive for cross-drive backup of this app
|
||||
for _, db := range backupDrives {
|
||||
rsyncBase := AppSecondaryRsyncPath(db.drivePath, stack.Name)
|
||||
if !dirExists(rsyncBase) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Found a backup for this app
|
||||
app.DrivePath = db.drivePath
|
||||
app.DriveLabel = db.label
|
||||
|
||||
// Check for _config/ (stack compose directory backup)
|
||||
configDir := filepath.Join(rsyncBase, "_config")
|
||||
if dirExists(configDir) {
|
||||
app.HasConfig = true
|
||||
app.ConfigPath = configDir
|
||||
}
|
||||
|
||||
// Check for _db/ (database dump backup)
|
||||
dbDir := filepath.Join(rsyncBase, "_db")
|
||||
if dirExists(dbDir) && !dirIsEmpty(dbDir) {
|
||||
app.HasDBDump = true
|
||||
app.DBDumpPath = dbDir
|
||||
}
|
||||
|
||||
// Check for user data in rsync (anything besides _config and _db)
|
||||
if hasUserData(rsyncBase) {
|
||||
app.HasRsyncData = true
|
||||
app.RsyncDataPath = rsyncBase
|
||||
}
|
||||
|
||||
break // use first drive with backup for this app
|
||||
}
|
||||
|
||||
plan.Apps = append(plan.Apps, app)
|
||||
}
|
||||
|
||||
if len(plan.Apps) == 0 {
|
||||
plan.Apps = []RestorableApp{}
|
||||
}
|
||||
|
||||
logger.Printf("[INFO] Restore plan: %d apps, %d drives (%d with backups)",
|
||||
len(plan.Apps), len(plan.Drives), len(backupDrives))
|
||||
|
||||
return plan
|
||||
}
|
||||
|
||||
// dirExists checks if a directory exists and is accessible.
|
||||
func dirExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
|
||||
// dirIsEmpty returns true if a directory has no entries.
|
||||
func dirIsEmpty(path string) bool {
|
||||
entries, err := os.ReadDir(path)
|
||||
return err != nil || len(entries) == 0
|
||||
}
|
||||
|
||||
// hasUserData checks if the rsync backup dir has user data (not just _config/_db).
|
||||
func hasUserData(rsyncBase string) bool {
|
||||
entries, err := os.ReadDir(rsyncBase)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if name != "_config" && name != "_db" && !hasPrefix(name, ".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasPrefix(s, prefix string) bool {
|
||||
return len(s) >= len(prefix) && s[:len(prefix)] == prefix
|
||||
}
|
||||
Reference in New Issue
Block a user