Files
felhom-controller/controller/internal/storage/migrate_drive.go
T
admin 95c821deb2 feat: comprehensive debug logging across all controller modules
Add detailed [DEBUG] logging to every controller module when
logging.level is set to "debug". Each module with stateful debug
uses SetDebug(bool) wired from main.go. Covers stacks, backup,
cloudflare, integrations, system, monitor, settings, scheduler,
web handlers, storage, metrics, API, selfupdate, and assets.

Also includes the app export/import (.fab bundles) feature from
v0.32.0 and its debug page integration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 18:14:43 +01:00

538 lines
16 KiB
Go

package storage
import (
"bufio"
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// StackProviderForMigration abstracts stack operations needed by drive migration.
type StackProviderForMigration interface {
ListDeployedStacks() []StackSummaryForMigration
GetStackHDDPath(name string) string
StopStack(name string) error
StartStack(name string) error
UpdateStackHDDPath(name, newPath string) error
StackExists(name string) bool
}
// StackSummaryForMigration holds minimal stack info for drive migration.
type StackSummaryForMigration struct {
Name string
DisplayName string
}
// DriveMigrateRequest holds parameters for migrating all apps from one drive to another.
type DriveMigrateRequest struct {
SourcePath string // e.g., "/mnt/hdd_1"
DestPath string // e.g., "/mnt/hdd_2"
}
// DriveMigrateProgress tracks drive migration state.
type DriveMigrateProgress struct {
Step string // "validating","stopping","copying","verifying","configuring","starting","backup","done","error","rolling_back"
Message string
BytesCopied int64
BytesTotal int64
Percent int
Error string
ElapsedSeconds int
Detail string // sub-step detail (e.g., which app is being configured)
}
// DriveMigrator orchestrates full drive migration.
type DriveMigrator struct {
Sett *settings.Settings
StackProvider StackProviderForMigration
BackupTrigger BackupTrigger
AlertRefresh func()
PushHubReport func()
PushInfraBackup func()
SyncFBMounts func()
Logger *log.Logger
mu sync.Mutex
active bool // global migration lock
}
// IsActive returns whether a full drive migration is currently in progress.
func (dm *DriveMigrator) IsActive() bool {
dm.mu.Lock()
defer dm.mu.Unlock()
return dm.active
}
// rollbackAction describes a reversible action in the migration transaction.
type rollbackAction struct {
description string
undo func() error
}
// migrationTx is a transaction log that enables reverse-order rollback.
type migrationTx struct {
actions []rollbackAction
logger *log.Logger
}
func (tx *migrationTx) add(desc string, undoFn func() error) {
tx.actions = append(tx.actions, rollbackAction{description: desc, undo: undoFn})
}
func (tx *migrationTx) rollback() {
for i := len(tx.actions) - 1; i >= 0; i-- {
a := tx.actions[i]
tx.logger.Printf("[ROLLBACK] %s", a.description)
if err := a.undo(); err != nil {
tx.logger.Printf("[ROLLBACK-ERROR] %s: %v", a.description, err)
}
}
}
// MigrateDrive performs a full drive migration, moving all apps from source to dest.
func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateRequest, progress chan<- DriveMigrateProgress) error {
start := time.Now()
debug := dm.Logger != nil
dbg := func(format string, args ...interface{}) {
if debug {
dm.Logger.Printf("[DEBUG] [storage] MigrateDrive: "+format, args...)
}
}
_ = dbg // used below
dbg("starting drive migration: source=%s dest=%s", req.SourcePath, req.DestPath)
send := func(step, msg string, pct int) {
progress <- DriveMigrateProgress{
Step: step,
Message: msg,
Percent: pct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
sendDetail := func(step, msg, detail string, pct int) {
progress <- DriveMigrateProgress{
Step: step,
Message: msg,
Detail: detail,
Percent: pct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
fail := func(msg string, err error) error {
errStr := ""
if err != nil {
errStr = err.Error()
}
progress <- DriveMigrateProgress{
Step: "error",
Message: msg,
Error: errStr,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
return fmt.Errorf("%s: %w", msg, err)
}
// Acquire global migration lock
dm.mu.Lock()
if dm.active {
dm.mu.Unlock()
return fail("Egy másik meghajtó-migráció folyamatban van", fmt.Errorf("migration already active"))
}
dm.active = true
dm.mu.Unlock()
defer func() {
dm.mu.Lock()
dm.active = false
dm.mu.Unlock()
}()
// --- Pre-validation ---
send("validating", "Ellenőrzés...", 1)
srcLabel := dm.Sett.GetStorageLabel(req.SourcePath)
dstLabel := dm.Sett.GetStorageLabel(req.DestPath)
if dm.Sett.IsDisconnected(req.SourcePath) {
return fail("A forrás meghajtó le van választva", fmt.Errorf("source disconnected"))
}
if dm.Sett.IsDecommissioned(req.SourcePath) {
return fail("A forrás meghajtó már kiváltott", fmt.Errorf("source decommissioned"))
}
if dm.Sett.IsDisconnected(req.DestPath) {
return fail("A cél meghajtó le van választva", fmt.Errorf("dest disconnected"))
}
if dm.Sett.IsDecommissioned(req.DestPath) {
return fail("A cél meghajtó már kiváltott", fmt.Errorf("dest decommissioned"))
}
// Find apps on source drive
var appsToMigrate []StackSummaryForMigration
for _, stack := range dm.StackProvider.ListDeployedStacks() {
hddPath := dm.StackProvider.GetStackHDDPath(stack.Name)
if hddPath == req.SourcePath {
appsToMigrate = append(appsToMigrate, stack)
}
}
dbg("found %d apps on source drive: %v", len(appsToMigrate), func() []string {
names := make([]string, len(appsToMigrate))
for i, a := range appsToMigrate {
names[i] = a.Name
}
return names
}())
if len(appsToMigrate) == 0 {
return fail("A forrás meghajtón nincs telepített alkalmazás", fmt.Errorf("no apps on source"))
}
// Check for conflicts on destination
for _, app := range appsToMigrate {
destAppData := backup.AppDataDir(req.DestPath, app.Name)
if info, err := os.Stat(destAppData); err == nil && info.IsDir() {
entries, _ := os.ReadDir(destAppData)
if len(entries) > 0 {
return fail(
fmt.Sprintf("A cél meghajtón már létezik adat: %s/%s", req.DestPath, app.Name),
fmt.Errorf("conflict: %s already exists on destination", app.Name),
)
}
}
}
// Estimate total size (exclude restic repos inside felhom-data/backups/)
var totalBytes int64
entries, _ := os.ReadDir(req.SourcePath)
for _, entry := range entries {
if !entry.IsDir() {
continue
}
entryPath := filepath.Join(req.SourcePath, entry.Name())
if entry.Name() == backup.FelhomDataDir {
// Scan inside namespace dir, excluding restic repos from estimate
subEntries, _ := os.ReadDir(entryPath)
for _, sub := range subEntries {
if !sub.IsDir() {
continue
}
subPath := filepath.Join(entryPath, sub.Name())
if sub.Name() == "backups" {
totalBytes += dirSizeExcluding(subPath, "restic")
} else {
totalBytes += dirSize(subPath)
}
}
} else {
totalBytes += dirSize(entryPath)
}
}
// Check free space on destination
freeBytes := getFreeBytes(req.DestPath)
if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes {
return fail(
fmt.Sprintf("Nincs elég szabad hely: szükséges ~%s, szabad %s",
bytesHuman(totalBytes), bytesHuman(freeBytes)),
fmt.Errorf("insufficient disk space"),
)
}
dbg("estimated data: %s (%d bytes), free on dest: %s (%d bytes)", bytesHuman(totalBytes), totalBytes, bytesHuman(freeBytes), freeBytes)
dm.Logger.Printf("[INFO] Drive migration: %s (%s) → %s (%s), %d apps, ~%s data",
req.SourcePath, srcLabel, req.DestPath, dstLabel, len(appsToMigrate), bytesHuman(totalBytes))
tx := &migrationTx{logger: dm.Logger}
// --- Step 1: Stop all affected apps ---
send("stopping", fmt.Sprintf("Alkalmazások leállítása (%d db)...", len(appsToMigrate)), 5)
var stoppedApps []string
for _, app := range appsToMigrate {
sendDetail("stopping", "Leállítás: "+app.DisplayName, app.Name, 5)
if err := dm.StackProvider.StopStack(app.Name); err != nil {
dm.Logger.Printf("[ERROR] Drive migration: failed to stop %s: %v", app.Name, err)
// Rollback: restart already stopped apps
send("rolling_back", "Hiba a leállításnál, visszagörgetés...", 0)
for _, name := range stoppedApps {
_ = dm.StackProvider.StartStack(name)
}
return fail("Alkalmazás leállítása sikertelen: "+app.DisplayName, err)
}
stoppedApps = append(stoppedApps, app.Name)
}
tx.add("Restart all stopped apps", func() error {
for _, name := range stoppedApps {
if err := dm.StackProvider.StartStack(name); err != nil {
dm.Logger.Printf("[ROLLBACK-WARN] Failed to restart %s: %v", name, err)
}
}
return nil
})
// --- Step 2: rsync entire drive (excluding restic repos) ---
send("copying", "Adatok másolása...", 10)
rsyncCmd := exec.CommandContext(ctx, "rsync", "-a", "--info=progress2",
"--exclude=felhom-data/backups/primary/restic/",
"--exclude=felhom-data/backups/secondary/restic/",
req.SourcePath+"/", req.DestPath+"/",
)
stdout, err := rsyncCmd.StdoutPipe()
if err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync pipe hiba", err)
}
stderr, err := rsyncCmd.StderrPipe()
if err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync stderr pipe hiba", err)
}
if err := rsyncCmd.Start(); err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync indítás sikertelen", err)
}
// Parse rsync progress
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
if b, pct, ok := parseRsyncProgress(line); ok {
scaledPct := 10 + pct*50/100 // scale to 10-60%
if scaledPct > 60 {
scaledPct = 60
}
progress <- DriveMigrateProgress{
Step: "copying",
Message: fmt.Sprintf("Adatok másolása... %s / %s", bytesHuman(b), bytesHuman(totalBytes)),
BytesCopied: b,
BytesTotal: totalBytes,
Percent: scaledPct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
}
}()
var stderrBuf strings.Builder
var stderrWg sync.WaitGroup
stderrWg.Add(1)
go func() {
defer stderrWg.Done()
buf := make([]byte, 4096)
for {
n, err := stderr.Read(buf)
if n > 0 {
stderrBuf.Write(buf[:n])
}
if err != nil {
break
}
}
}()
if err := rsyncCmd.Wait(); err != nil {
stderrWg.Wait()
dbg("rsync failed after %s: %v — stderr: %s", time.Since(start).Round(time.Second), err, stderrBuf.String())
send("rolling_back", "rsync sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("Adatmásolás sikertelen", fmt.Errorf("rsync failed: %w — %s", err, stderrBuf.String()))
}
stderrWg.Wait()
dbg("rsync completed in %s", time.Since(start).Round(time.Second))
// --- Step 3: Verify copy ---
send("verifying", "Másolat ellenőrzése...", 62)
for _, app := range appsToMigrate {
destAppData := backup.AppDataDir(req.DestPath, app.Name)
if _, err := os.Stat(destAppData); os.IsNotExist(err) {
// appdata might not exist for all apps (SSD-only apps that share the drive)
// Only warn, don't fail
dm.Logger.Printf("[WARN] Drive migration: %s not found on destination (may be SSD-only)", destAppData)
}
}
// --- Step 4: Update all app configs ---
send("configuring", "Konfiguráció frissítése...", 65)
dbg("updating HDD_PATH for %d apps", len(appsToMigrate))
var configuredApps []string
for i, app := range appsToMigrate {
// Guard: verify app still exists
if !dm.StackProvider.StackExists(app.Name) {
dm.Logger.Printf("[WARN] Drive migration: app %s no longer exists, skipping config update", app.Name)
continue
}
pct := 65 + (i * 10 / len(appsToMigrate))
sendDetail("configuring", "Konfiguráció: "+app.DisplayName, app.Name, pct)
oldPath := dm.StackProvider.GetStackHDDPath(app.Name)
if err := dm.StackProvider.UpdateStackHDDPath(app.Name, req.DestPath); err != nil {
dm.Logger.Printf("[ERROR] Drive migration: failed to update HDD_PATH for %s: %v", app.Name, err)
send("rolling_back", "Konfiguráció frissítése sikertelen, visszagörgetés...", 0)
// Rollback config changes
for _, name := range configuredApps {
_ = dm.StackProvider.UpdateStackHDDPath(name, req.SourcePath)
}
tx.rollback()
return fail("HDD_PATH frissítés sikertelen: "+app.DisplayName, err)
}
configuredApps = append(configuredApps, app.Name)
tx.add("Revert HDD_PATH for "+app.Name, func() error {
return dm.StackProvider.UpdateStackHDDPath(app.Name, oldPath)
})
}
// --- Step 5: Update storage registry ---
send("configuring", "Tárolóregiszter frissítése...", 76)
// Transfer IsDefault
allPaths := dm.Sett.GetStoragePaths()
var srcWasDefault bool
var srcWasSchedulable bool
for _, sp := range allPaths {
if sp.Path == req.SourcePath {
srcWasDefault = sp.IsDefault
srcWasSchedulable = sp.Schedulable
}
}
if srcWasDefault {
_ = dm.Sett.SetDefaultStoragePath(req.DestPath)
}
if srcWasSchedulable {
_ = dm.Sett.SetSchedulable(req.DestPath, true)
}
// Mark source as decommissioned
if err := dm.Sett.SetDecommissioned(req.SourcePath, req.DestPath); err != nil {
dm.Logger.Printf("[WARN] Drive migration: failed to mark source as decommissioned: %v", err)
}
tx.add("Clear decommissioned on source", func() error {
return dm.Sett.ClearDecommissioned(req.SourcePath)
})
// --- Step 6: Update Tier 2 cross-drive configs ---
send("configuring", "Mentési beállítások frissítése...", 78)
allCrossConfigs := dm.Sett.GetAllCrossDriveConfigs()
for name, cfg := range allCrossConfigs {
if cfg == nil {
continue
}
// Apps that moved (source→dest) with Tier 2 pointing to dest: clear (no redundancy)
appHDD := dm.StackProvider.GetStackHDDPath(name)
if appHDD == req.DestPath && cfg.DestinationPath == req.DestPath {
dm.Logger.Printf("[INFO] Drive migration: clearing Tier 2 for %s (dest same as app drive)", name)
_ = dm.Sett.SetCrossDriveConfig(name, nil)
continue
}
// Apps on OTHER drives with Tier 2 pointing to source: redirect to dest
if cfg.DestinationPath == req.SourcePath {
dm.Logger.Printf("[INFO] Drive migration: redirecting Tier 2 for %s from %s to %s", name, req.SourcePath, req.DestPath)
cfg.DestinationPath = req.DestPath
_ = dm.Sett.SetCrossDriveConfig(name, cfg)
}
}
// --- Step 7: Start all migrated apps ---
send("starting", "Alkalmazások indítása...", 80)
for i, app := range appsToMigrate {
if !dm.StackProvider.StackExists(app.Name) {
continue
}
pct := 80 + (i * 8 / len(appsToMigrate))
sendDetail("starting", "Indítás: "+app.DisplayName, app.Name, pct)
if err := dm.StackProvider.StartStack(app.Name); err != nil {
dm.Logger.Printf("[WARN] Drive migration: failed to start %s after migration: %v", app.Name, err)
// Non-fatal — log but continue
}
}
// At this point, migration is considered successful — no more rollback.
// --- Step 8: Trigger immediate backup ---
send("backup", "Biztonsági mentés indítása...", 90)
if dm.BackupTrigger != nil {
if err := dm.BackupTrigger.TryRunDriveBackup(ctx, req.DestPath); err != nil {
dm.Logger.Printf("[WARN] Drive migration: post-migration backup failed: %v", err)
}
}
// --- Step 9: Post-migration notifications ---
send("configuring", "Befejező lépések...", 95)
if dm.SyncFBMounts != nil {
dm.SyncFBMounts()
}
if dm.AlertRefresh != nil {
dm.AlertRefresh()
}
if dm.PushHubReport != nil {
dm.PushHubReport()
}
if dm.PushInfraBackup != nil {
dm.PushInfraBackup()
}
elapsed := time.Since(start)
dm.Logger.Printf("[INFO] Drive migration complete: %s → %s, %d apps, %s elapsed",
req.SourcePath, req.DestPath, len(appsToMigrate), elapsed.Round(time.Second))
// --- Step 10: Done ---
appNames := make([]string, len(appsToMigrate))
for i, app := range appsToMigrate {
appNames[i] = app.DisplayName
}
progress <- DriveMigrateProgress{
Step: "done",
Message: fmt.Sprintf("A %s meghajtó sikeresen kiváltva! %d alkalmazás átköltöztetve ide: %s (%s). Idő: %s",
srcLabel, len(appsToMigrate), dstLabel, req.DestPath, elapsed.Round(time.Second)),
Percent: 100,
ElapsedSeconds: int(elapsed.Seconds()),
Detail: strings.Join(appNames, ", "),
}
return nil
}
// dirSizeExcluding returns the total bytes in a directory, excluding subdirectories named excludeName.
func dirSizeExcluding(path, excludeName string) int64 {
var total int64
filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if info.IsDir() && info.Name() == excludeName {
return filepath.SkipDir
}
if !info.IsDir() {
total += info.Size()
}
return nil
})
return total
}