feat: drive migration & Tier 2 restic deprecation (v0.18.0)
Phase 1: Deprecate restic as Tier 2 method (rsync only), auto-migrate on startup Phase 2: Enhanced per-app migration with backup awareness, DB dump copy, auto-cleanup Phase 3: Full drive migration with decommissioned state, rollback support, wizard UI Phase 4: Hub report includes decommissioned drive state Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,497 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// StackProviderForMigration abstracts stack operations needed by drive migration.
|
||||
type StackProviderForMigration interface {
|
||||
ListDeployedStacks() []StackSummaryForMigration
|
||||
GetStackHDDPath(name string) string
|
||||
StopStack(name string) error
|
||||
StartStack(name string) error
|
||||
UpdateStackHDDPath(name, newPath string) error
|
||||
StackExists(name string) bool
|
||||
}
|
||||
|
||||
// StackSummaryForMigration holds minimal stack info for drive migration.
|
||||
type StackSummaryForMigration struct {
|
||||
Name string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
// DriveMigrateRequest holds parameters for migrating all apps from one drive to another.
|
||||
type DriveMigrateRequest struct {
|
||||
SourcePath string // e.g., "/mnt/hdd_1"
|
||||
DestPath string // e.g., "/mnt/hdd_2"
|
||||
}
|
||||
|
||||
// DriveMigrateProgress tracks drive migration state.
|
||||
type DriveMigrateProgress struct {
|
||||
Step string // "validating","stopping","copying","verifying","configuring","starting","backup","done","error","rolling_back"
|
||||
Message string
|
||||
BytesCopied int64
|
||||
BytesTotal int64
|
||||
Percent int
|
||||
Error string
|
||||
ElapsedSeconds int
|
||||
Detail string // sub-step detail (e.g., which app is being configured)
|
||||
}
|
||||
|
||||
// DriveMigrator orchestrates full drive migration.
|
||||
type DriveMigrator struct {
|
||||
Sett *settings.Settings
|
||||
StackProvider StackProviderForMigration
|
||||
BackupTrigger BackupTrigger
|
||||
AlertRefresh func()
|
||||
PushHubReport func()
|
||||
PushInfraBackup func()
|
||||
SyncFBMounts func()
|
||||
Logger *log.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
active bool // global migration lock
|
||||
}
|
||||
|
||||
// IsActive returns whether a full drive migration is currently in progress.
|
||||
func (dm *DriveMigrator) IsActive() bool {
|
||||
dm.mu.Lock()
|
||||
defer dm.mu.Unlock()
|
||||
return dm.active
|
||||
}
|
||||
|
||||
// rollbackAction describes a reversible action in the migration transaction.
|
||||
type rollbackAction struct {
|
||||
description string
|
||||
undo func() error
|
||||
}
|
||||
|
||||
// migrationTx is a transaction log that enables reverse-order rollback.
|
||||
type migrationTx struct {
|
||||
actions []rollbackAction
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func (tx *migrationTx) add(desc string, undoFn func() error) {
|
||||
tx.actions = append(tx.actions, rollbackAction{description: desc, undo: undoFn})
|
||||
}
|
||||
|
||||
func (tx *migrationTx) rollback() {
|
||||
for i := len(tx.actions) - 1; i >= 0; i-- {
|
||||
a := tx.actions[i]
|
||||
tx.logger.Printf("[ROLLBACK] %s", a.description)
|
||||
if err := a.undo(); err != nil {
|
||||
tx.logger.Printf("[ROLLBACK-ERROR] %s: %v", a.description, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MigrateDrive performs a full drive migration, moving all apps from source to dest.
|
||||
func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateRequest, progress chan<- DriveMigrateProgress) error {
|
||||
start := time.Now()
|
||||
|
||||
send := func(step, msg string, pct int) {
|
||||
progress <- DriveMigrateProgress{
|
||||
Step: step,
|
||||
Message: msg,
|
||||
Percent: pct,
|
||||
ElapsedSeconds: int(time.Since(start).Seconds()),
|
||||
}
|
||||
}
|
||||
sendDetail := func(step, msg, detail string, pct int) {
|
||||
progress <- DriveMigrateProgress{
|
||||
Step: step,
|
||||
Message: msg,
|
||||
Detail: detail,
|
||||
Percent: pct,
|
||||
ElapsedSeconds: int(time.Since(start).Seconds()),
|
||||
}
|
||||
}
|
||||
fail := func(msg string, err error) error {
|
||||
errStr := ""
|
||||
if err != nil {
|
||||
errStr = err.Error()
|
||||
}
|
||||
progress <- DriveMigrateProgress{
|
||||
Step: "error",
|
||||
Message: msg,
|
||||
Error: errStr,
|
||||
ElapsedSeconds: int(time.Since(start).Seconds()),
|
||||
}
|
||||
return fmt.Errorf("%s: %w", msg, err)
|
||||
}
|
||||
|
||||
// Acquire global migration lock
|
||||
dm.mu.Lock()
|
||||
if dm.active {
|
||||
dm.mu.Unlock()
|
||||
return fail("Egy másik meghajtó-migráció folyamatban van", fmt.Errorf("migration already active"))
|
||||
}
|
||||
dm.active = true
|
||||
dm.mu.Unlock()
|
||||
defer func() {
|
||||
dm.mu.Lock()
|
||||
dm.active = false
|
||||
dm.mu.Unlock()
|
||||
}()
|
||||
|
||||
// --- Pre-validation ---
|
||||
send("validating", "Ellenőrzés...", 1)
|
||||
|
||||
srcLabel := dm.Sett.GetStorageLabel(req.SourcePath)
|
||||
dstLabel := dm.Sett.GetStorageLabel(req.DestPath)
|
||||
|
||||
if dm.Sett.IsDisconnected(req.SourcePath) {
|
||||
return fail("A forrás meghajtó le van választva", fmt.Errorf("source disconnected"))
|
||||
}
|
||||
if dm.Sett.IsDecommissioned(req.SourcePath) {
|
||||
return fail("A forrás meghajtó már kiváltott", fmt.Errorf("source decommissioned"))
|
||||
}
|
||||
if dm.Sett.IsDisconnected(req.DestPath) {
|
||||
return fail("A cél meghajtó le van választva", fmt.Errorf("dest disconnected"))
|
||||
}
|
||||
if dm.Sett.IsDecommissioned(req.DestPath) {
|
||||
return fail("A cél meghajtó már kiváltott", fmt.Errorf("dest decommissioned"))
|
||||
}
|
||||
|
||||
// Find apps on source drive
|
||||
var appsToMigrate []StackSummaryForMigration
|
||||
for _, stack := range dm.StackProvider.ListDeployedStacks() {
|
||||
hddPath := dm.StackProvider.GetStackHDDPath(stack.Name)
|
||||
if hddPath == req.SourcePath {
|
||||
appsToMigrate = append(appsToMigrate, stack)
|
||||
}
|
||||
}
|
||||
|
||||
if len(appsToMigrate) == 0 {
|
||||
return fail("A forrás meghajtón nincs telepített alkalmazás", fmt.Errorf("no apps on source"))
|
||||
}
|
||||
|
||||
// Check for conflicts on destination
|
||||
for _, app := range appsToMigrate {
|
||||
destAppData := filepath.Join(req.DestPath, "appdata", app.Name)
|
||||
if info, err := os.Stat(destAppData); err == nil && info.IsDir() {
|
||||
entries, _ := os.ReadDir(destAppData)
|
||||
if len(entries) > 0 {
|
||||
return fail(
|
||||
fmt.Sprintf("A cél meghajtón már létezik adat: %s/%s", req.DestPath, app.Name),
|
||||
fmt.Errorf("conflict: %s already exists on destination", app.Name),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate total size (exclude restic repos)
|
||||
var totalBytes int64
|
||||
entries, _ := os.ReadDir(req.SourcePath)
|
||||
for _, entry := range entries {
|
||||
entryPath := filepath.Join(req.SourcePath, entry.Name())
|
||||
if entry.IsDir() {
|
||||
// Skip restic repos in size estimate
|
||||
if entry.Name() == "backups" {
|
||||
totalBytes += dirSizeExcluding(entryPath, "restic")
|
||||
} else {
|
||||
totalBytes += dirSize(entryPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check free space on destination
|
||||
freeBytes := getFreeBytes(req.DestPath)
|
||||
if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes {
|
||||
return fail(
|
||||
fmt.Sprintf("Nincs elég szabad hely: szükséges ~%s, szabad %s",
|
||||
bytesHuman(totalBytes), bytesHuman(freeBytes)),
|
||||
fmt.Errorf("insufficient disk space"),
|
||||
)
|
||||
}
|
||||
|
||||
dm.Logger.Printf("[INFO] Drive migration: %s (%s) → %s (%s), %d apps, ~%s data",
|
||||
req.SourcePath, srcLabel, req.DestPath, dstLabel, len(appsToMigrate), bytesHuman(totalBytes))
|
||||
|
||||
tx := &migrationTx{logger: dm.Logger}
|
||||
|
||||
// --- Step 1: Stop all affected apps ---
|
||||
send("stopping", fmt.Sprintf("Alkalmazások leállítása (%d db)...", len(appsToMigrate)), 5)
|
||||
|
||||
var stoppedApps []string
|
||||
for _, app := range appsToMigrate {
|
||||
sendDetail("stopping", "Leállítás: "+app.DisplayName, app.Name, 5)
|
||||
if err := dm.StackProvider.StopStack(app.Name); err != nil {
|
||||
dm.Logger.Printf("[ERROR] Drive migration: failed to stop %s: %v", app.Name, err)
|
||||
// Rollback: restart already stopped apps
|
||||
send("rolling_back", "Hiba a leállításnál, visszagörgetés...", 0)
|
||||
for _, name := range stoppedApps {
|
||||
_ = dm.StackProvider.StartStack(name)
|
||||
}
|
||||
return fail("Alkalmazás leállítása sikertelen: "+app.DisplayName, err)
|
||||
}
|
||||
stoppedApps = append(stoppedApps, app.Name)
|
||||
}
|
||||
tx.add("Restart all stopped apps", func() error {
|
||||
for _, name := range stoppedApps {
|
||||
if err := dm.StackProvider.StartStack(name); err != nil {
|
||||
dm.Logger.Printf("[ROLLBACK-WARN] Failed to restart %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// --- Step 2: rsync entire drive (excluding restic repos) ---
|
||||
send("copying", "Adatok másolása...", 10)
|
||||
|
||||
rsyncCmd := exec.CommandContext(ctx, "rsync", "-a", "--info=progress2",
|
||||
"--exclude=backups/primary/restic/",
|
||||
"--exclude=backups/secondary/restic/",
|
||||
req.SourcePath+"/", req.DestPath+"/",
|
||||
)
|
||||
|
||||
stdout, err := rsyncCmd.StdoutPipe()
|
||||
if err != nil {
|
||||
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
|
||||
tx.rollback()
|
||||
return fail("rsync pipe hiba", err)
|
||||
}
|
||||
stderr, err := rsyncCmd.StderrPipe()
|
||||
if err != nil {
|
||||
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
|
||||
tx.rollback()
|
||||
return fail("rsync stderr pipe hiba", err)
|
||||
}
|
||||
|
||||
if err := rsyncCmd.Start(); err != nil {
|
||||
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
|
||||
tx.rollback()
|
||||
return fail("rsync indítás sikertelen", err)
|
||||
}
|
||||
|
||||
// Parse rsync progress
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if b, pct, ok := parseRsyncProgress(line); ok {
|
||||
scaledPct := 10 + pct*50/100 // scale to 10-60%
|
||||
if scaledPct > 60 {
|
||||
scaledPct = 60
|
||||
}
|
||||
progress <- DriveMigrateProgress{
|
||||
Step: "copying",
|
||||
Message: fmt.Sprintf("Adatok másolása... %s / %s", bytesHuman(b), bytesHuman(totalBytes)),
|
||||
BytesCopied: b,
|
||||
BytesTotal: totalBytes,
|
||||
Percent: scaledPct,
|
||||
ElapsedSeconds: int(time.Since(start).Seconds()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var stderrBuf strings.Builder
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := stderr.Read(buf)
|
||||
if n > 0 {
|
||||
stderrBuf.Write(buf[:n])
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err := rsyncCmd.Wait(); err != nil {
|
||||
send("rolling_back", "rsync sikertelen, visszagörgetés...", 0)
|
||||
tx.rollback()
|
||||
return fail("Adatmásolás sikertelen", fmt.Errorf("rsync failed: %w — %s", err, stderrBuf.String()))
|
||||
}
|
||||
|
||||
// --- Step 3: Verify copy ---
|
||||
send("verifying", "Másolat ellenőrzése...", 62)
|
||||
|
||||
for _, app := range appsToMigrate {
|
||||
destAppData := filepath.Join(req.DestPath, "appdata", app.Name)
|
||||
if _, err := os.Stat(destAppData); os.IsNotExist(err) {
|
||||
// appdata might not exist for all apps (SSD-only apps that share the drive)
|
||||
// Only warn, don't fail
|
||||
dm.Logger.Printf("[WARN] Drive migration: %s/appdata/%s not found on destination (may be SSD-only)", req.DestPath, app.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 4: Update all app configs ---
|
||||
send("configuring", "Konfiguráció frissítése...", 65)
|
||||
|
||||
var configuredApps []string
|
||||
for i, app := range appsToMigrate {
|
||||
// Guard: verify app still exists
|
||||
if !dm.StackProvider.StackExists(app.Name) {
|
||||
dm.Logger.Printf("[WARN] Drive migration: app %s no longer exists, skipping config update", app.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
pct := 65 + (i * 10 / len(appsToMigrate))
|
||||
sendDetail("configuring", "Konfiguráció: "+app.DisplayName, app.Name, pct)
|
||||
|
||||
oldPath := dm.StackProvider.GetStackHDDPath(app.Name)
|
||||
if err := dm.StackProvider.UpdateStackHDDPath(app.Name, req.DestPath); err != nil {
|
||||
dm.Logger.Printf("[ERROR] Drive migration: failed to update HDD_PATH for %s: %v", app.Name, err)
|
||||
send("rolling_back", "Konfiguráció frissítése sikertelen, visszagörgetés...", 0)
|
||||
// Rollback config changes
|
||||
for _, name := range configuredApps {
|
||||
_ = dm.StackProvider.UpdateStackHDDPath(name, req.SourcePath)
|
||||
}
|
||||
tx.rollback()
|
||||
return fail("HDD_PATH frissítés sikertelen: "+app.DisplayName, err)
|
||||
}
|
||||
configuredApps = append(configuredApps, app.Name)
|
||||
tx.add("Revert HDD_PATH for "+app.Name, func() error {
|
||||
return dm.StackProvider.UpdateStackHDDPath(app.Name, oldPath)
|
||||
})
|
||||
}
|
||||
|
||||
// --- Step 5: Update storage registry ---
|
||||
send("configuring", "Tárolóregiszter frissítése...", 76)
|
||||
|
||||
// Transfer IsDefault
|
||||
allPaths := dm.Sett.GetStoragePaths()
|
||||
var srcWasDefault bool
|
||||
var srcWasSchedulable bool
|
||||
for _, sp := range allPaths {
|
||||
if sp.Path == req.SourcePath {
|
||||
srcWasDefault = sp.IsDefault
|
||||
srcWasSchedulable = sp.Schedulable
|
||||
}
|
||||
}
|
||||
|
||||
if srcWasDefault {
|
||||
_ = dm.Sett.SetDefaultStoragePath(req.DestPath)
|
||||
}
|
||||
if srcWasSchedulable {
|
||||
_ = dm.Sett.SetSchedulable(req.DestPath, true)
|
||||
}
|
||||
|
||||
// Mark source as decommissioned
|
||||
if err := dm.Sett.SetDecommissioned(req.SourcePath, req.DestPath); err != nil {
|
||||
dm.Logger.Printf("[WARN] Drive migration: failed to mark source as decommissioned: %v", err)
|
||||
}
|
||||
tx.add("Clear decommissioned on source", func() error {
|
||||
return dm.Sett.ClearDecommissioned(req.SourcePath)
|
||||
})
|
||||
|
||||
// --- Step 6: Update Tier 2 cross-drive configs ---
|
||||
send("configuring", "Mentési beállítások frissítése...", 78)
|
||||
|
||||
allCrossConfigs := dm.Sett.GetAllCrossDriveConfigs()
|
||||
for name, cfg := range allCrossConfigs {
|
||||
if cfg == nil {
|
||||
continue
|
||||
}
|
||||
// Apps that moved (source→dest) with Tier 2 pointing to dest: clear (no redundancy)
|
||||
appHDD := dm.StackProvider.GetStackHDDPath(name)
|
||||
if appHDD == req.DestPath && cfg.DestinationPath == req.DestPath {
|
||||
dm.Logger.Printf("[INFO] Drive migration: clearing Tier 2 for %s (dest same as app drive)", name)
|
||||
_ = dm.Sett.SetCrossDriveConfig(name, nil)
|
||||
continue
|
||||
}
|
||||
// Apps on OTHER drives with Tier 2 pointing to source: redirect to dest
|
||||
if cfg.DestinationPath == req.SourcePath {
|
||||
dm.Logger.Printf("[INFO] Drive migration: redirecting Tier 2 for %s from %s to %s", name, req.SourcePath, req.DestPath)
|
||||
cfg.DestinationPath = req.DestPath
|
||||
_ = dm.Sett.SetCrossDriveConfig(name, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 7: Start all migrated apps ---
|
||||
send("starting", "Alkalmazások indítása...", 80)
|
||||
|
||||
for i, app := range appsToMigrate {
|
||||
if !dm.StackProvider.StackExists(app.Name) {
|
||||
continue
|
||||
}
|
||||
pct := 80 + (i * 8 / len(appsToMigrate))
|
||||
sendDetail("starting", "Indítás: "+app.DisplayName, app.Name, pct)
|
||||
|
||||
if err := dm.StackProvider.StartStack(app.Name); err != nil {
|
||||
dm.Logger.Printf("[WARN] Drive migration: failed to start %s after migration: %v", app.Name, err)
|
||||
// Non-fatal — log but continue
|
||||
}
|
||||
}
|
||||
|
||||
// At this point, migration is considered successful — no more rollback.
|
||||
|
||||
// --- Step 8: Trigger immediate backup ---
|
||||
send("backup", "Biztonsági mentés indítása...", 90)
|
||||
|
||||
if dm.BackupTrigger != nil {
|
||||
if err := dm.BackupTrigger.TryRunDriveBackup(ctx, req.DestPath); err != nil {
|
||||
dm.Logger.Printf("[WARN] Drive migration: post-migration backup failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 9: Post-migration notifications ---
|
||||
send("configuring", "Befejező lépések...", 95)
|
||||
|
||||
if dm.SyncFBMounts != nil {
|
||||
dm.SyncFBMounts()
|
||||
}
|
||||
if dm.AlertRefresh != nil {
|
||||
dm.AlertRefresh()
|
||||
}
|
||||
if dm.PushHubReport != nil {
|
||||
dm.PushHubReport()
|
||||
}
|
||||
if dm.PushInfraBackup != nil {
|
||||
dm.PushInfraBackup()
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
dm.Logger.Printf("[INFO] Drive migration complete: %s → %s, %d apps, %s elapsed",
|
||||
req.SourcePath, req.DestPath, len(appsToMigrate), elapsed.Round(time.Second))
|
||||
|
||||
// --- Step 10: Done ---
|
||||
appNames := make([]string, len(appsToMigrate))
|
||||
for i, app := range appsToMigrate {
|
||||
appNames[i] = app.DisplayName
|
||||
}
|
||||
|
||||
progress <- DriveMigrateProgress{
|
||||
Step: "done",
|
||||
Message: fmt.Sprintf("A %s meghajtó sikeresen kiváltva! %d alkalmazás átköltöztetve ide: %s (%s). Idő: %s",
|
||||
srcLabel, len(appsToMigrate), dstLabel, req.DestPath, elapsed.Round(time.Second)),
|
||||
Percent: 100,
|
||||
ElapsedSeconds: int(elapsed.Seconds()),
|
||||
Detail: strings.Join(appNames, ", "),
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// dirSizeExcluding returns the total bytes in a directory, excluding subdirectories named excludeName.
|
||||
func dirSizeExcluding(path, excludeName string) int64 {
|
||||
var total int64
|
||||
filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() && info.Name() == excludeName {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return total
|
||||
}
|
||||
Reference in New Issue
Block a user