package storage import ( "bufio" "context" "fmt" "log" "os" "os/exec" "path/filepath" "strings" "sync" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/backup" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // StackProviderForMigration abstracts stack operations needed by drive migration. type StackProviderForMigration interface { ListDeployedStacks() []StackSummaryForMigration GetStackHDDPath(name string) string StopStack(name string) error StartStack(name string) error UpdateStackHDDPath(name, newPath string) error StackExists(name string) bool } // StackSummaryForMigration holds minimal stack info for drive migration. type StackSummaryForMigration struct { Name string DisplayName string } // DriveMigrateRequest holds parameters for migrating all apps from one drive to another. type DriveMigrateRequest struct { SourcePath string // e.g., "/mnt/hdd_1" DestPath string // e.g., "/mnt/hdd_2" } // DriveMigrateProgress tracks drive migration state. type DriveMigrateProgress struct { Step string // "validating","stopping","copying","verifying","configuring","starting","backup","done","error","rolling_back" Message string BytesCopied int64 BytesTotal int64 Percent int Error string ElapsedSeconds int Detail string // sub-step detail (e.g., which app is being configured) } // DriveMigrator orchestrates full drive migration. type DriveMigrator struct { Sett *settings.Settings StackProvider StackProviderForMigration BackupTrigger BackupTrigger AlertRefresh func() PushHubReport func() PushInfraBackup func() SyncFBMounts func() Logger *log.Logger mu sync.Mutex active bool // global migration lock } // IsActive returns whether a full drive migration is currently in progress. func (dm *DriveMigrator) IsActive() bool { dm.mu.Lock() defer dm.mu.Unlock() return dm.active } // rollbackAction describes a reversible action in the migration transaction. type rollbackAction struct { description string undo func() error } // migrationTx is a transaction log that enables reverse-order rollback. type migrationTx struct { actions []rollbackAction logger *log.Logger } func (tx *migrationTx) add(desc string, undoFn func() error) { tx.actions = append(tx.actions, rollbackAction{description: desc, undo: undoFn}) } func (tx *migrationTx) rollback() { for i := len(tx.actions) - 1; i >= 0; i-- { a := tx.actions[i] tx.logger.Printf("[INFO] [storage] Rollback: %s", a.description) if err := a.undo(); err != nil { tx.logger.Printf("[ERROR] [storage] Rollback failed: %s: %v", a.description, err) } } } // MigrateDrive performs a full drive migration, moving all apps from source to dest. func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateRequest, progress chan<- DriveMigrateProgress) error { start := time.Now() // TODO: debug should be driven by a dedicated Debug field, not Logger presence. // Currently always true when Logger is set (which is always in practice). debug := dm.Logger != nil dbg := func(format string, args ...interface{}) { if debug { dm.Logger.Printf("[DEBUG] [storage] MigrateDrive: "+format, args...) } } _ = dbg // used below dbg("starting drive migration: source=%s dest=%s", req.SourcePath, req.DestPath) send := func(step, msg string, pct int) { progress <- DriveMigrateProgress{ Step: step, Message: msg, Percent: pct, ElapsedSeconds: int(time.Since(start).Seconds()), } } sendDetail := func(step, msg, detail string, pct int) { progress <- DriveMigrateProgress{ Step: step, Message: msg, Detail: detail, Percent: pct, ElapsedSeconds: int(time.Since(start).Seconds()), } } fail := func(msg string, err error) error { errStr := "" if err != nil { errStr = err.Error() } progress <- DriveMigrateProgress{ Step: "error", Message: msg, Error: errStr, ElapsedSeconds: int(time.Since(start).Seconds()), } return fmt.Errorf("%s: %w", msg, err) } // Acquire global migration lock dm.mu.Lock() if dm.active { dm.mu.Unlock() return fail("Egy másik meghajtó-migráció folyamatban van", fmt.Errorf("migration already active")) } dm.active = true dm.mu.Unlock() defer func() { dm.mu.Lock() dm.active = false dm.mu.Unlock() }() // --- Pre-validation --- send("validating", "Ellenőrzés...", 1) srcLabel := dm.Sett.GetStorageLabel(req.SourcePath) dstLabel := dm.Sett.GetStorageLabel(req.DestPath) if dm.Sett.IsDisconnected(req.SourcePath) { return fail("A forrás meghajtó le van választva", fmt.Errorf("source disconnected")) } if dm.Sett.IsDecommissioned(req.SourcePath) { return fail("A forrás meghajtó már kiváltott", fmt.Errorf("source decommissioned")) } if dm.Sett.IsDisconnected(req.DestPath) { return fail("A cél meghajtó le van választva", fmt.Errorf("dest disconnected")) } if dm.Sett.IsDecommissioned(req.DestPath) { return fail("A cél meghajtó már kiváltott", fmt.Errorf("dest decommissioned")) } // Find apps on source drive var appsToMigrate []StackSummaryForMigration for _, stack := range dm.StackProvider.ListDeployedStacks() { hddPath := dm.StackProvider.GetStackHDDPath(stack.Name) if hddPath == req.SourcePath { appsToMigrate = append(appsToMigrate, stack) } } dbg("found %d apps on source drive: %v", len(appsToMigrate), func() []string { names := make([]string, len(appsToMigrate)) for i, a := range appsToMigrate { names[i] = a.Name } return names }()) if len(appsToMigrate) == 0 { return fail("A forrás meghajtón nincs telepített alkalmazás", fmt.Errorf("no apps on source")) } // Check for conflicts on destination for _, app := range appsToMigrate { destAppData := backup.AppDataDir(req.DestPath, app.Name) if info, err := os.Stat(destAppData); err == nil && info.IsDir() { entries, _ := os.ReadDir(destAppData) if len(entries) > 0 { return fail( fmt.Sprintf("A cél meghajtón már létezik adat: %s/%s", req.DestPath, app.Name), fmt.Errorf("conflict: %s already exists on destination", app.Name), ) } } } // Estimate total size (exclude restic repos inside felhom-data/backups/) var totalBytes int64 entries, _ := os.ReadDir(req.SourcePath) for _, entry := range entries { if !entry.IsDir() { continue } entryPath := filepath.Join(req.SourcePath, entry.Name()) if entry.Name() == backup.FelhomDataDir { // Scan inside namespace dir, excluding restic repos from estimate subEntries, _ := os.ReadDir(entryPath) for _, sub := range subEntries { if !sub.IsDir() { continue } subPath := filepath.Join(entryPath, sub.Name()) if sub.Name() == "backups" { totalBytes += dirSizeExcluding(subPath, "restic") } else { totalBytes += dirSize(subPath) } } } else { totalBytes += dirSize(entryPath) } } // Check free space on destination freeBytes := getFreeBytes(req.DestPath) if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes { return fail( fmt.Sprintf("Nincs elég szabad hely: szükséges ~%s, szabad %s", bytesHuman(totalBytes), bytesHuman(freeBytes)), fmt.Errorf("insufficient disk space"), ) } dbg("estimated data: %s (%d bytes), free on dest: %s (%d bytes)", bytesHuman(totalBytes), totalBytes, bytesHuman(freeBytes), freeBytes) dm.Logger.Printf("[INFO] [storage] Drive migration: %s (%s) → %s (%s), %d apps, ~%s data", req.SourcePath, srcLabel, req.DestPath, dstLabel, len(appsToMigrate), bytesHuman(totalBytes)) tx := &migrationTx{logger: dm.Logger} // --- Step 1: Stop all affected apps --- send("stopping", fmt.Sprintf("Alkalmazások leállítása (%d db)...", len(appsToMigrate)), 5) var stoppedApps []string for _, app := range appsToMigrate { sendDetail("stopping", "Leállítás: "+app.DisplayName, app.Name, 5) if err := dm.StackProvider.StopStack(app.Name); err != nil { dm.Logger.Printf("[ERROR] [storage] Drive migration: failed to stop %s: %v", app.Name, err) // Rollback: restart already stopped apps send("rolling_back", "Hiba a leállításnál, visszagörgetés...", 0) for _, name := range stoppedApps { _ = dm.StackProvider.StartStack(name) } return fail("Alkalmazás leállítása sikertelen: "+app.DisplayName, err) } stoppedApps = append(stoppedApps, app.Name) } tx.add("Restart all stopped apps", func() error { for _, name := range stoppedApps { if err := dm.StackProvider.StartStack(name); err != nil { dm.Logger.Printf("[WARN] [storage] Rollback: failed to restart %s: %v", name, err) } } return nil }) // --- Step 2: rsync entire drive (excluding restic repos) --- send("copying", "Adatok másolása...", 10) rsyncCmd := exec.CommandContext(ctx, "rsync", "-a", "--info=progress2", "--exclude=felhom-data/backups/primary/restic/", "--exclude=felhom-data/backups/secondary/restic/", req.SourcePath+"/", req.DestPath+"/", ) stdout, err := rsyncCmd.StdoutPipe() if err != nil { send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0) tx.rollback() return fail("rsync pipe hiba", err) } stderr, err := rsyncCmd.StderrPipe() if err != nil { send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0) tx.rollback() return fail("rsync stderr pipe hiba", err) } if err := rsyncCmd.Start(); err != nil { send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0) tx.rollback() return fail("rsync indítás sikertelen", err) } // Parse rsync progress go func() { scanner := bufio.NewScanner(stdout) for scanner.Scan() { line := scanner.Text() if b, pct, ok := parseRsyncProgress(line); ok { scaledPct := 10 + pct*50/100 // scale to 10-60% if scaledPct > 60 { scaledPct = 60 } progress <- DriveMigrateProgress{ Step: "copying", Message: fmt.Sprintf("Adatok másolása... %s / %s", bytesHuman(b), bytesHuman(totalBytes)), BytesCopied: b, BytesTotal: totalBytes, Percent: scaledPct, ElapsedSeconds: int(time.Since(start).Seconds()), } } } }() var stderrBuf strings.Builder var stderrWg sync.WaitGroup stderrWg.Add(1) go func() { defer stderrWg.Done() buf := make([]byte, 4096) for { n, err := stderr.Read(buf) if n > 0 { stderrBuf.Write(buf[:n]) } if err != nil { break } } }() if err := rsyncCmd.Wait(); err != nil { stderrWg.Wait() dbg("rsync failed after %s: %v — stderr: %s", time.Since(start).Round(time.Second), err, stderrBuf.String()) send("rolling_back", "rsync sikertelen, visszagörgetés...", 0) tx.rollback() return fail("Adatmásolás sikertelen", fmt.Errorf("rsync failed: %w — %s", err, stderrBuf.String())) } stderrWg.Wait() dbg("rsync completed in %s", time.Since(start).Round(time.Second)) // --- Step 3: Verify copy --- send("verifying", "Másolat ellenőrzése...", 62) for _, app := range appsToMigrate { destAppData := backup.AppDataDir(req.DestPath, app.Name) if _, err := os.Stat(destAppData); os.IsNotExist(err) { // appdata might not exist for all apps (SSD-only apps that share the drive) // Only warn, don't fail dm.Logger.Printf("[WARN] [storage] Drive migration: %s not found on destination (may be SSD-only)", destAppData) } } // --- Step 4: Update all app configs --- send("configuring", "Konfiguráció frissítése...", 65) dbg("updating HDD_PATH for %d apps", len(appsToMigrate)) var configuredApps []string for i, app := range appsToMigrate { // Guard: verify app still exists if !dm.StackProvider.StackExists(app.Name) { dm.Logger.Printf("[WARN] [storage] Drive migration: app %s no longer exists, skipping config update", app.Name) continue } pct := 65 + (i * 10 / len(appsToMigrate)) sendDetail("configuring", "Konfiguráció: "+app.DisplayName, app.Name, pct) oldPath := dm.StackProvider.GetStackHDDPath(app.Name) if err := dm.StackProvider.UpdateStackHDDPath(app.Name, req.DestPath); err != nil { dm.Logger.Printf("[ERROR] [storage] Drive migration: failed to update HDD_PATH for %s: %v", app.Name, err) send("rolling_back", "Konfiguráció frissítése sikertelen, visszagörgetés...", 0) // Rollback config changes for _, name := range configuredApps { _ = dm.StackProvider.UpdateStackHDDPath(name, req.SourcePath) } tx.rollback() return fail("HDD_PATH frissítés sikertelen: "+app.DisplayName, err) } configuredApps = append(configuredApps, app.Name) tx.add("Revert HDD_PATH for "+app.Name, func() error { return dm.StackProvider.UpdateStackHDDPath(app.Name, oldPath) }) } // --- Step 5: Update storage registry --- send("configuring", "Tárolóregiszter frissítése...", 76) // Transfer IsDefault allPaths := dm.Sett.GetStoragePaths() var srcWasDefault bool var srcWasSchedulable bool for _, sp := range allPaths { if sp.Path == req.SourcePath { srcWasDefault = sp.IsDefault srcWasSchedulable = sp.Schedulable } } if srcWasDefault { _ = dm.Sett.SetDefaultStoragePath(req.DestPath) } if srcWasSchedulable { _ = dm.Sett.SetSchedulable(req.DestPath, true) } // Mark source as decommissioned if err := dm.Sett.SetDecommissioned(req.SourcePath, req.DestPath); err != nil { dm.Logger.Printf("[WARN] [storage] Drive migration: failed to mark source as decommissioned: %v", err) } tx.add("Clear decommissioned on source", func() error { return dm.Sett.ClearDecommissioned(req.SourcePath) }) // --- Step 6: Update Tier 2 cross-drive configs --- send("configuring", "Mentési beállítások frissítése...", 78) allCrossConfigs := dm.Sett.GetAllCrossDriveConfigs() for name, cfg := range allCrossConfigs { if cfg == nil { continue } // Apps that moved (source→dest) with Tier 2 pointing to dest: clear (no redundancy) appHDD := dm.StackProvider.GetStackHDDPath(name) if appHDD == req.DestPath && cfg.DestinationPath == req.DestPath { dm.Logger.Printf("[INFO] [storage] Drive migration: clearing Tier 2 for %s (dest same as app drive)", name) _ = dm.Sett.SetCrossDriveConfig(name, nil) continue } // Apps on OTHER drives with Tier 2 pointing to source: redirect to dest if cfg.DestinationPath == req.SourcePath { dm.Logger.Printf("[INFO] [storage] Drive migration: redirecting Tier 2 for %s from %s to %s", name, req.SourcePath, req.DestPath) cfg.DestinationPath = req.DestPath _ = dm.Sett.SetCrossDriveConfig(name, cfg) } } // --- Step 7: Start all migrated apps --- send("starting", "Alkalmazások indítása...", 80) for i, app := range appsToMigrate { if !dm.StackProvider.StackExists(app.Name) { continue } pct := 80 + (i * 8 / len(appsToMigrate)) sendDetail("starting", "Indítás: "+app.DisplayName, app.Name, pct) if err := dm.StackProvider.StartStack(app.Name); err != nil { dm.Logger.Printf("[WARN] [storage] Drive migration: failed to start %s after migration: %v", app.Name, err) // Non-fatal — log but continue } } // At this point, migration is considered successful — no more rollback. // --- Step 8: Trigger immediate backup --- send("backup", "Biztonsági mentés indítása...", 90) if dm.BackupTrigger != nil { if err := dm.BackupTrigger.TryRunDriveBackup(ctx, req.DestPath); err != nil { dm.Logger.Printf("[WARN] [storage] Drive migration: post-migration backup failed: %v", err) } } // --- Step 9: Post-migration notifications --- send("configuring", "Befejező lépések...", 95) if dm.SyncFBMounts != nil { dm.SyncFBMounts() } if dm.AlertRefresh != nil { dm.AlertRefresh() } if dm.PushHubReport != nil { dm.PushHubReport() } if dm.PushInfraBackup != nil { dm.PushInfraBackup() } elapsed := time.Since(start) dm.Logger.Printf("[INFO] [storage] Drive migration complete: %s → %s, %d apps, %s elapsed", req.SourcePath, req.DestPath, len(appsToMigrate), elapsed.Round(time.Second)) // --- Step 10: Done --- appNames := make([]string, len(appsToMigrate)) for i, app := range appsToMigrate { appNames[i] = app.DisplayName } progress <- DriveMigrateProgress{ Step: "done", Message: fmt.Sprintf("A %s meghajtó sikeresen kiváltva! %d alkalmazás átköltöztetve ide: %s (%s). Idő: %s", srcLabel, len(appsToMigrate), dstLabel, req.DestPath, elapsed.Round(time.Second)), Percent: 100, ElapsedSeconds: int(elapsed.Seconds()), Detail: strings.Join(appNames, ", "), } return nil } // dirSizeExcluding returns the total bytes in a directory, excluding subdirectories named excludeName. func dirSizeExcluding(path, excludeName string) int64 { var total int64 filepath.Walk(path, func(p string, info os.FileInfo, err error) error { if err != nil { return nil } if info.IsDir() && info.Name() == excludeName { return filepath.SkipDir } if !info.IsDir() { total += info.Size() } return nil }) return total }