Files
deploy-felhom-compose/controller/internal/backup/restore_scan.go
T
admin 95c821deb2 feat: comprehensive debug logging across all controller modules
Add detailed [DEBUG] logging to every controller module when
logging.level is set to "debug". Each module with stateful debug
uses SetDebug(bool) wired from main.go. Covers stacks, backup,
cloudflare, integrations, system, monitor, settings, scheduler,
web handlers, storage, metrics, API, selfupdate, and assets.

Also includes the app export/import (.fab bundles) feature from
v0.32.0 and its debug page integration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 18:14:43 +01:00

311 lines
8.9 KiB
Go

package backup
import (
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
// RestorableApp describes an app that can be restored during DR.
type RestorableApp struct {
Name string `json:"name"`
DisplayName string `json:"display_name"`
NeedsHDD bool `json:"needs_hdd"`
HDDPath string `json:"hdd_path,omitempty"`
// What was found on disk
HasConfig bool `json:"has_config"` // _config/ dir with compose files
ConfigPath string `json:"config_path"` // full path to _config/ backup
HasData bool `json:"has_data"` // app data dir exists on HDD
DataPath string `json:"data_path"` // e.g., /mnt/hdd_1/appdata/immich
HasDBDump bool `json:"has_db_dump"` // _db/ dir with dump files
DBDumpPath string `json:"db_dump_path"` // full path to _db/ backup
HasRsyncData bool `json:"has_rsync_data"` // rsync user data (excl _config/_db)
RsyncDataPath string `json:"rsync_data_path"` // full path to rsync backup
DrivePath string `json:"drive_path"` // which drive has the backup
DriveLabel string `json:"drive_label"` // label for display
// Restore progress (updated during restore)
Status string `json:"status"` // "pending", "restoring", "done", "failed", "skipped"
Error string `json:"error,omitempty"`
StartedAt string `json:"started_at,omitempty"`
CompletedAt string `json:"completed_at,omitempty"`
}
// RestorePlan holds the complete DR restore plan.
type RestorePlan struct {
mu sync.RWMutex
CustomerID string `json:"customer_id"`
Domain string `json:"domain"`
Timestamp string `json:"timestamp"` // when the infra backup was taken
Apps []RestorableApp `json:"apps"`
// Drive summary
Drives []DriveInfo `json:"drives"`
// Overall status
Status string `json:"status"` // "pending", "restoring", "done"
}
// DriveInfo summarizes a mounted drive for display.
type DriveInfo struct {
Path string `json:"path"`
Label string `json:"label"`
Available bool `json:"available"` // mount is accessible
HasBackup bool `json:"has_backup"` // has backups/secondary/ dir
}
// GetApps returns a snapshot of the apps list.
func (rp *RestorePlan) GetApps() []RestorableApp {
rp.mu.RLock()
defer rp.mu.RUnlock()
apps := make([]RestorableApp, len(rp.Apps))
copy(apps, rp.Apps)
return apps
}
// Snapshot returns a thread-safe snapshot of the plan for JSON serialization.
func (rp *RestorePlan) Snapshot() map[string]interface{} {
rp.mu.RLock()
defer rp.mu.RUnlock()
apps := make([]RestorableApp, len(rp.Apps))
copy(apps, rp.Apps)
drives := make([]DriveInfo, len(rp.Drives))
copy(drives, rp.Drives)
return map[string]interface{}{
"ok": true,
"status": rp.Status,
"apps": apps,
"drives": drives,
}
}
// TryStartRestore atomically sets status to "restoring" if not already restoring.
// Returns false if a restore is already in progress (prevents double-restore race).
func (rp *RestorePlan) TryStartRestore() bool {
rp.mu.Lock()
defer rp.mu.Unlock()
if rp.Status == "restoring" {
return false
}
rp.Status = "restoring"
return true
}
// SetStatus sets the overall plan status under lock.
func (rp *RestorePlan) SetStatus(status string) {
rp.mu.Lock()
defer rp.mu.Unlock()
rp.Status = status
}
// GetStatus returns the current plan status under lock.
func (rp *RestorePlan) GetStatus() string {
rp.mu.RLock()
defer rp.mu.RUnlock()
return rp.Status
}
// UpdateApp updates a single app's status in the plan.
func (rp *RestorePlan) UpdateApp(name, status, errMsg string) {
rp.mu.Lock()
defer rp.mu.Unlock()
for i := range rp.Apps {
if rp.Apps[i].Name == name {
rp.Apps[i].Status = status
rp.Apps[i].Error = errMsg
if status == "restoring" {
rp.Apps[i].StartedAt = time.Now().UTC().Format(time.RFC3339)
}
if status == "done" || status == "failed" {
rp.Apps[i].CompletedAt = time.Now().UTC().Format(time.RFC3339)
}
return
}
}
}
// AllDone returns true if all apps are done/failed/skipped.
// Returns false for empty plans (no apps to restore).
func (rp *RestorePlan) AllDone() bool {
rp.mu.RLock()
defer rp.mu.RUnlock()
if len(rp.Apps) == 0 {
return false
}
for _, app := range rp.Apps {
if app.Status != "done" && app.Status != "failed" && app.Status != "skipped" {
return false
}
}
return true
}
// InfraStackInfo is a minimal stack descriptor from the Hub infra backup.
// Used to pass deployed_stacks info into the scan without importing report.
type InfraStackInfo struct {
Name string
DisplayName string
HDDPath string
NeedsHDD bool
}
// ScanDrivesForBackups scans mounted drives for cross-drive backup data
// and correlates with the deployed stacks manifest from the Hub.
func ScanDrivesForBackups(mountedPaths []string, stacks []InfraStackInfo, logger *log.Logger) *RestorePlan {
plan := &RestorePlan{
Status: "pending",
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning %d mount paths, %d stacks from manifest",
len(mountedPaths), len(stacks))
// Build drive info and find backup directories
type driveBackup struct {
drivePath string
label string
secPath string // backups/secondary/ path
}
var backupDrives []driveBackup
for _, mp := range mountedPaths {
label := filepath.Base(mp)
avail := dirExists(mp)
di := DriveInfo{
Path: mp,
Label: label,
Available: avail,
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: checking drive %s (label=%s, available=%v)", mp, label, avail)
secPath := SecondaryBackupPath(mp)
if dirExists(secPath) {
di.HasBackup = true
backupDrives = append(backupDrives, driveBackup{
drivePath: mp,
label: label,
secPath: secPath,
})
logger.Printf("[INFO] Found backup data on %s (%s)", mp, secPath)
}
plan.Drives = append(plan.Drives, di)
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: found %d drives with backup data", len(backupDrives))
// For each stack from the manifest, look for backup data on drives
for _, stack := range stacks {
app := RestorableApp{
Name: stack.Name,
DisplayName: stack.DisplayName,
NeedsHDD: stack.NeedsHDD,
HDDPath: stack.HDDPath,
Status: "pending",
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning for app %s (needsHDD=%v, hddPath=%s)",
stack.Name, stack.NeedsHDD, stack.HDDPath)
// Check if app data exists directly on HDD (common case: HDD survived)
if stack.HDDPath != "" {
dataDir := AppDataDir(stack.HDDPath, stack.Name)
if dirExists(dataDir) {
app.HasData = true
app.DataPath = dataDir
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — live data found at %s", stack.Name, dataDir)
}
}
// Scan each drive for cross-drive backup of this app
for _, db := range backupDrives {
rsyncBase := AppSecondaryRsyncPath(db.drivePath, stack.Name)
if !dirExists(rsyncBase) {
continue
}
// Found a backup for this app
app.DrivePath = db.drivePath
app.DriveLabel = db.label
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — backup found on drive %s at %s",
stack.Name, db.label, rsyncBase)
// Check for _config/ (stack compose directory backup)
configDir := filepath.Join(rsyncBase, "_config")
if dirExists(configDir) {
app.HasConfig = true
app.ConfigPath = configDir
}
// Check for _db/ (database dump backup)
dbDir := filepath.Join(rsyncBase, "_db")
if dirExists(dbDir) && !dirIsEmpty(dbDir) {
app.HasDBDump = true
app.DBDumpPath = dbDir
}
// Check for user data in rsync (anything besides _config and _db)
if hasUserData(rsyncBase) {
app.HasRsyncData = true
app.RsyncDataPath = rsyncBase
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — config=%v, dbDump=%v, rsyncData=%v",
stack.Name, app.HasConfig, app.HasDBDump, app.HasRsyncData)
break // use first drive with backup for this app
}
plan.Apps = append(plan.Apps, app)
}
if len(plan.Apps) == 0 {
plan.Apps = []RestorableApp{}
}
logger.Printf("[INFO] Restore plan: %d apps, %d drives (%d with backups)",
len(plan.Apps), len(plan.Drives), len(backupDrives))
return plan
}
// dirExists checks if a directory exists and is accessible.
func dirExists(path string) bool {
info, err := os.Stat(path)
return err == nil && info.IsDir()
}
// dirIsEmpty returns true if a directory has no entries.
// Returns false on read errors (assume non-empty — safer for backup detection).
func dirIsEmpty(path string) bool {
entries, err := os.ReadDir(path)
if err != nil {
return false
}
return len(entries) == 0
}
// hasUserData checks if the rsync backup dir has user data (not just _config/_db).
func hasUserData(rsyncBase string) bool {
entries, err := os.ReadDir(rsyncBase)
if err != nil {
return false
}
for _, e := range entries {
name := e.Name()
if name != "_config" && name != "_db" && !strings.HasPrefix(name, ".") {
return true
}
}
return false
}