95c821deb2
Add detailed [DEBUG] logging to every controller module when logging.level is set to "debug". Each module with stateful debug uses SetDebug(bool) wired from main.go. Covers stacks, backup, cloudflare, integrations, system, monitor, settings, scheduler, web handlers, storage, metrics, API, selfupdate, and assets. Also includes the app export/import (.fab bundles) feature from v0.32.0 and its debug page integration. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
311 lines
8.9 KiB
Go
311 lines
8.9 KiB
Go
package backup
|
|
|
|
import (
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// RestorableApp describes an app that can be restored during DR.
|
|
type RestorableApp struct {
|
|
Name string `json:"name"`
|
|
DisplayName string `json:"display_name"`
|
|
NeedsHDD bool `json:"needs_hdd"`
|
|
HDDPath string `json:"hdd_path,omitempty"`
|
|
|
|
// What was found on disk
|
|
HasConfig bool `json:"has_config"` // _config/ dir with compose files
|
|
ConfigPath string `json:"config_path"` // full path to _config/ backup
|
|
HasData bool `json:"has_data"` // app data dir exists on HDD
|
|
DataPath string `json:"data_path"` // e.g., /mnt/hdd_1/appdata/immich
|
|
HasDBDump bool `json:"has_db_dump"` // _db/ dir with dump files
|
|
DBDumpPath string `json:"db_dump_path"` // full path to _db/ backup
|
|
HasRsyncData bool `json:"has_rsync_data"` // rsync user data (excl _config/_db)
|
|
RsyncDataPath string `json:"rsync_data_path"` // full path to rsync backup
|
|
DrivePath string `json:"drive_path"` // which drive has the backup
|
|
DriveLabel string `json:"drive_label"` // label for display
|
|
|
|
// Restore progress (updated during restore)
|
|
Status string `json:"status"` // "pending", "restoring", "done", "failed", "skipped"
|
|
Error string `json:"error,omitempty"`
|
|
StartedAt string `json:"started_at,omitempty"`
|
|
CompletedAt string `json:"completed_at,omitempty"`
|
|
}
|
|
|
|
// RestorePlan holds the complete DR restore plan.
|
|
type RestorePlan struct {
|
|
mu sync.RWMutex
|
|
|
|
CustomerID string `json:"customer_id"`
|
|
Domain string `json:"domain"`
|
|
Timestamp string `json:"timestamp"` // when the infra backup was taken
|
|
Apps []RestorableApp `json:"apps"`
|
|
|
|
// Drive summary
|
|
Drives []DriveInfo `json:"drives"`
|
|
|
|
// Overall status
|
|
Status string `json:"status"` // "pending", "restoring", "done"
|
|
}
|
|
|
|
// DriveInfo summarizes a mounted drive for display.
|
|
type DriveInfo struct {
|
|
Path string `json:"path"`
|
|
Label string `json:"label"`
|
|
Available bool `json:"available"` // mount is accessible
|
|
HasBackup bool `json:"has_backup"` // has backups/secondary/ dir
|
|
}
|
|
|
|
// GetApps returns a snapshot of the apps list.
|
|
func (rp *RestorePlan) GetApps() []RestorableApp {
|
|
rp.mu.RLock()
|
|
defer rp.mu.RUnlock()
|
|
apps := make([]RestorableApp, len(rp.Apps))
|
|
copy(apps, rp.Apps)
|
|
return apps
|
|
}
|
|
|
|
// Snapshot returns a thread-safe snapshot of the plan for JSON serialization.
|
|
func (rp *RestorePlan) Snapshot() map[string]interface{} {
|
|
rp.mu.RLock()
|
|
defer rp.mu.RUnlock()
|
|
|
|
apps := make([]RestorableApp, len(rp.Apps))
|
|
copy(apps, rp.Apps)
|
|
drives := make([]DriveInfo, len(rp.Drives))
|
|
copy(drives, rp.Drives)
|
|
|
|
return map[string]interface{}{
|
|
"ok": true,
|
|
"status": rp.Status,
|
|
"apps": apps,
|
|
"drives": drives,
|
|
}
|
|
}
|
|
|
|
// TryStartRestore atomically sets status to "restoring" if not already restoring.
|
|
// Returns false if a restore is already in progress (prevents double-restore race).
|
|
func (rp *RestorePlan) TryStartRestore() bool {
|
|
rp.mu.Lock()
|
|
defer rp.mu.Unlock()
|
|
if rp.Status == "restoring" {
|
|
return false
|
|
}
|
|
rp.Status = "restoring"
|
|
return true
|
|
}
|
|
|
|
// SetStatus sets the overall plan status under lock.
|
|
func (rp *RestorePlan) SetStatus(status string) {
|
|
rp.mu.Lock()
|
|
defer rp.mu.Unlock()
|
|
rp.Status = status
|
|
}
|
|
|
|
// GetStatus returns the current plan status under lock.
|
|
func (rp *RestorePlan) GetStatus() string {
|
|
rp.mu.RLock()
|
|
defer rp.mu.RUnlock()
|
|
return rp.Status
|
|
}
|
|
|
|
// UpdateApp updates a single app's status in the plan.
|
|
func (rp *RestorePlan) UpdateApp(name, status, errMsg string) {
|
|
rp.mu.Lock()
|
|
defer rp.mu.Unlock()
|
|
for i := range rp.Apps {
|
|
if rp.Apps[i].Name == name {
|
|
rp.Apps[i].Status = status
|
|
rp.Apps[i].Error = errMsg
|
|
if status == "restoring" {
|
|
rp.Apps[i].StartedAt = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
if status == "done" || status == "failed" {
|
|
rp.Apps[i].CompletedAt = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// AllDone returns true if all apps are done/failed/skipped.
|
|
// Returns false for empty plans (no apps to restore).
|
|
func (rp *RestorePlan) AllDone() bool {
|
|
rp.mu.RLock()
|
|
defer rp.mu.RUnlock()
|
|
if len(rp.Apps) == 0 {
|
|
return false
|
|
}
|
|
for _, app := range rp.Apps {
|
|
if app.Status != "done" && app.Status != "failed" && app.Status != "skipped" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// InfraStackInfo is a minimal stack descriptor from the Hub infra backup.
|
|
// Used to pass deployed_stacks info into the scan without importing report.
|
|
type InfraStackInfo struct {
|
|
Name string
|
|
DisplayName string
|
|
HDDPath string
|
|
NeedsHDD bool
|
|
}
|
|
|
|
// ScanDrivesForBackups scans mounted drives for cross-drive backup data
|
|
// and correlates with the deployed stacks manifest from the Hub.
|
|
func ScanDrivesForBackups(mountedPaths []string, stacks []InfraStackInfo, logger *log.Logger) *RestorePlan {
|
|
plan := &RestorePlan{
|
|
Status: "pending",
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning %d mount paths, %d stacks from manifest",
|
|
len(mountedPaths), len(stacks))
|
|
|
|
// Build drive info and find backup directories
|
|
type driveBackup struct {
|
|
drivePath string
|
|
label string
|
|
secPath string // backups/secondary/ path
|
|
}
|
|
var backupDrives []driveBackup
|
|
|
|
for _, mp := range mountedPaths {
|
|
label := filepath.Base(mp)
|
|
avail := dirExists(mp)
|
|
|
|
di := DriveInfo{
|
|
Path: mp,
|
|
Label: label,
|
|
Available: avail,
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: checking drive %s (label=%s, available=%v)", mp, label, avail)
|
|
|
|
secPath := SecondaryBackupPath(mp)
|
|
if dirExists(secPath) {
|
|
di.HasBackup = true
|
|
backupDrives = append(backupDrives, driveBackup{
|
|
drivePath: mp,
|
|
label: label,
|
|
secPath: secPath,
|
|
})
|
|
logger.Printf("[INFO] Found backup data on %s (%s)", mp, secPath)
|
|
}
|
|
|
|
plan.Drives = append(plan.Drives, di)
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: found %d drives with backup data", len(backupDrives))
|
|
|
|
// For each stack from the manifest, look for backup data on drives
|
|
for _, stack := range stacks {
|
|
app := RestorableApp{
|
|
Name: stack.Name,
|
|
DisplayName: stack.DisplayName,
|
|
NeedsHDD: stack.NeedsHDD,
|
|
HDDPath: stack.HDDPath,
|
|
Status: "pending",
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning for app %s (needsHDD=%v, hddPath=%s)",
|
|
stack.Name, stack.NeedsHDD, stack.HDDPath)
|
|
|
|
// Check if app data exists directly on HDD (common case: HDD survived)
|
|
if stack.HDDPath != "" {
|
|
dataDir := AppDataDir(stack.HDDPath, stack.Name)
|
|
if dirExists(dataDir) {
|
|
app.HasData = true
|
|
app.DataPath = dataDir
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — live data found at %s", stack.Name, dataDir)
|
|
}
|
|
}
|
|
|
|
// Scan each drive for cross-drive backup of this app
|
|
for _, db := range backupDrives {
|
|
rsyncBase := AppSecondaryRsyncPath(db.drivePath, stack.Name)
|
|
if !dirExists(rsyncBase) {
|
|
continue
|
|
}
|
|
|
|
// Found a backup for this app
|
|
app.DrivePath = db.drivePath
|
|
app.DriveLabel = db.label
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — backup found on drive %s at %s",
|
|
stack.Name, db.label, rsyncBase)
|
|
|
|
// Check for _config/ (stack compose directory backup)
|
|
configDir := filepath.Join(rsyncBase, "_config")
|
|
if dirExists(configDir) {
|
|
app.HasConfig = true
|
|
app.ConfigPath = configDir
|
|
}
|
|
|
|
// Check for _db/ (database dump backup)
|
|
dbDir := filepath.Join(rsyncBase, "_db")
|
|
if dirExists(dbDir) && !dirIsEmpty(dbDir) {
|
|
app.HasDBDump = true
|
|
app.DBDumpPath = dbDir
|
|
}
|
|
|
|
// Check for user data in rsync (anything besides _config and _db)
|
|
if hasUserData(rsyncBase) {
|
|
app.HasRsyncData = true
|
|
app.RsyncDataPath = rsyncBase
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — config=%v, dbDump=%v, rsyncData=%v",
|
|
stack.Name, app.HasConfig, app.HasDBDump, app.HasRsyncData)
|
|
|
|
break // use first drive with backup for this app
|
|
}
|
|
|
|
plan.Apps = append(plan.Apps, app)
|
|
}
|
|
|
|
if len(plan.Apps) == 0 {
|
|
plan.Apps = []RestorableApp{}
|
|
}
|
|
|
|
logger.Printf("[INFO] Restore plan: %d apps, %d drives (%d with backups)",
|
|
len(plan.Apps), len(plan.Drives), len(backupDrives))
|
|
|
|
return plan
|
|
}
|
|
|
|
// dirExists checks if a directory exists and is accessible.
|
|
func dirExists(path string) bool {
|
|
info, err := os.Stat(path)
|
|
return err == nil && info.IsDir()
|
|
}
|
|
|
|
// dirIsEmpty returns true if a directory has no entries.
|
|
// Returns false on read errors (assume non-empty — safer for backup detection).
|
|
func dirIsEmpty(path string) bool {
|
|
entries, err := os.ReadDir(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return len(entries) == 0
|
|
}
|
|
|
|
// hasUserData checks if the rsync backup dir has user data (not just _config/_db).
|
|
func hasUserData(rsyncBase string) bool {
|
|
entries, err := os.ReadDir(rsyncBase)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if name != "_config" && name != "_db" && !strings.HasPrefix(name, ".") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|