8e61cd7ec4
Add structured operational logging at INFO, WARN, and ERROR levels to every controller module. Standardize custom prefixes ([GEO], [SCHED], [SYNC]) to use [INFO/WARN/ERROR] [module] format. Fix misleveled logs (WARN->ERROR for data loss scenarios, WARN->INFO for routine operations). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1121 lines
32 KiB
Go
1121 lines
32 KiB
Go
package appexport
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ScannedBundle describes a .fab file found on a storage drive.
|
|
type ScannedBundle struct {
|
|
Path string `json:"path"`
|
|
FileName string `json:"file_name"`
|
|
AppName string `json:"app_name"`
|
|
DisplayName string `json:"display_name"`
|
|
ExportedAt string `json:"exported_at"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
SizeHuman string `json:"size_human"`
|
|
Encrypted bool `json:"encrypted"`
|
|
NeedsHDD bool `json:"needs_hdd"`
|
|
HasDB bool `json:"has_db"`
|
|
DrivePath string `json:"drive_path"`
|
|
DriveLabel string `json:"drive_label"`
|
|
}
|
|
|
|
// ImportRequest holds user-provided parameters for an import.
|
|
type ImportRequest struct {
|
|
FABPath string // full path to .fab file
|
|
Password string // empty for unencrypted bundles
|
|
}
|
|
|
|
// ScanForBundles scans export directories on all registered drives for .fab files.
|
|
// Pass a non-nil logger to enable debug output.
|
|
func ScanForBundles(drives []DrivePathInfo) []ScannedBundle {
|
|
var bundles []ScannedBundle
|
|
|
|
for _, drive := range drives {
|
|
exportDir := ExportDir(drive.Path)
|
|
entries, err := os.ReadDir(exportDir)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".fab") {
|
|
continue
|
|
}
|
|
|
|
path := filepath.Join(exportDir, entry.Name())
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
bundle := ScannedBundle{
|
|
Path: path,
|
|
FileName: entry.Name(),
|
|
SizeBytes: info.Size(),
|
|
SizeHuman: humanizeBytes(info.Size()),
|
|
DrivePath: drive.Path,
|
|
DriveLabel: drive.Label,
|
|
}
|
|
|
|
// Check encryption
|
|
encrypted, _ := IsEncryptedFAB(path)
|
|
bundle.Encrypted = encrypted
|
|
|
|
// Try to read manifest for metadata (unencrypted only)
|
|
var manifest *Manifest
|
|
if !encrypted {
|
|
manifest, _ = ReadManifestFromFAB(path)
|
|
}
|
|
|
|
if manifest != nil {
|
|
bundle.AppName = manifest.AppName
|
|
bundle.DisplayName = manifest.DisplayName
|
|
bundle.ExportedAt = manifest.ExportedAt.Format("2006-01-02 15:04")
|
|
bundle.NeedsHDD = manifest.NeedsHDD
|
|
bundle.HasDB = manifest.HasDatabase
|
|
} else {
|
|
// Parse app name from filename: {appname}_{timestamp}.fab
|
|
name := strings.TrimSuffix(entry.Name(), ".fab")
|
|
if idx := strings.LastIndex(name, "_"); idx > 0 {
|
|
bundle.AppName = name[:idx]
|
|
} else {
|
|
bundle.AppName = name
|
|
}
|
|
bundle.ExportedAt = info.ModTime().Format("2006-01-02 15:04")
|
|
}
|
|
|
|
bundles = append(bundles, bundle)
|
|
}
|
|
}
|
|
|
|
return bundles
|
|
}
|
|
|
|
// ScanForStaleTempFiles finds stale .fab.tmp and .tgz.tmp files in export directories.
|
|
func ScanForStaleTempFiles(drives []DrivePathInfo) []string {
|
|
var stale []string
|
|
for _, drive := range drives {
|
|
exportDir := ExportDir(drive.Path)
|
|
entries, err := os.ReadDir(exportDir)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
if strings.HasSuffix(name, ".tmp") || strings.HasSuffix(name, ".tgz.tmp") {
|
|
stale = append(stale, filepath.Join(exportDir, name))
|
|
}
|
|
}
|
|
}
|
|
return stale
|
|
}
|
|
|
|
// ReadManifestFromFAB reads the manifest from an unencrypted .fab file.
|
|
func ReadManifestFromFAB(fabPath string) (*Manifest, error) {
|
|
f, err := os.Open(fabPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("not a valid gzip: %w", err)
|
|
}
|
|
defer gr.Close()
|
|
|
|
tr := tar.NewReader(gr)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil, fmt.Errorf("manifest.json not found in bundle")
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if hdr.Name == "manifest.json" {
|
|
data, err := io.ReadAll(tr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return UnmarshalManifest(data)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ReadManifestFromEncryptedFAB decrypts and reads the manifest from an encrypted .fab file.
|
|
func ReadManifestFromEncryptedFAB(fabPath, password string) (*Manifest, error) {
|
|
tmpFile, err := os.CreateTemp("", "fab-manifest-*.tgz")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
tmpFile.Close()
|
|
defer os.Remove(tmpPath)
|
|
|
|
if err := DecryptFile(fabPath, tmpPath, password); err != nil {
|
|
return nil, fmt.Errorf("visszafejtés sikertelen: %w", err)
|
|
}
|
|
|
|
return ReadManifestFromFAB(tmpPath)
|
|
}
|
|
|
|
// StartImport validates and starts an async import. Returns error if blocked.
|
|
func (e *Exporter) StartImport(req ImportRequest) error {
|
|
e.mu.Lock()
|
|
if e.activeJob != nil && e.activeJob.Running {
|
|
e.mu.Unlock()
|
|
e.debugf("StartImport rejected: another job is already running")
|
|
return fmt.Errorf("export or import already in progress")
|
|
}
|
|
|
|
// Validate file exists
|
|
info, err := os.Stat(req.FABPath)
|
|
if err != nil {
|
|
e.mu.Unlock()
|
|
e.debugf("StartImport rejected: file not found: %s", req.FABPath)
|
|
return fmt.Errorf("bundle file not found: %w", err)
|
|
}
|
|
|
|
e.debugf("StartImport: path=%s size=%s encrypted=%v",
|
|
req.FABPath, humanizeBytes(info.Size()), req.Password != "")
|
|
|
|
steps := []Step{
|
|
{Label: "Csomag megnyitása", Status: "pending"},
|
|
{Label: "Alkalmazás előkészítése", Status: "pending"},
|
|
{Label: "Konfiguráció visszaállítása", Status: "pending"},
|
|
{Label: "Felhasználói adatok visszaállítása", Status: "pending"},
|
|
{Label: "Adatbázis visszaállítása", Status: "pending"},
|
|
{Label: "Alkalmazás indítása", Status: "pending"},
|
|
}
|
|
|
|
job := &Job{
|
|
StackName: filepath.Base(req.FABPath),
|
|
Steps: steps,
|
|
Running: true,
|
|
JobType: "import",
|
|
}
|
|
e.activeJob = job
|
|
e.mu.Unlock()
|
|
|
|
go e.executeImport(req, job)
|
|
return nil
|
|
}
|
|
|
|
func (e *Exporter) executeImport(req ImportRequest, job *Job) {
|
|
importStart := time.Now()
|
|
e.logger.Printf("[INFO] [appexport] Import started for %s", filepath.Base(req.FABPath))
|
|
e.debugf("=== IMPORT START: path=%s encrypted=%v ===", req.FABPath, req.Password != "")
|
|
|
|
defer func() {
|
|
job.mu.Lock()
|
|
job.Running = false
|
|
job.Done = true
|
|
job.mu.Unlock()
|
|
e.debugf("=== IMPORT END: elapsed=%v ===", time.Since(importStart))
|
|
}()
|
|
|
|
step := 0
|
|
|
|
// --- Step 0: Open bundle ---
|
|
job.setStep(step, "running", "")
|
|
stepStart := time.Now()
|
|
|
|
encrypted, _ := IsEncryptedFAB(req.FABPath)
|
|
e.debugf("bundle encrypted: %v", encrypted)
|
|
tgzPath := req.FABPath
|
|
|
|
// Decrypt if needed
|
|
if encrypted {
|
|
if req.Password == "" {
|
|
e.failJob(job, step, "Titkosított csomag — jelszó szükséges")
|
|
return
|
|
}
|
|
tmp, err := os.CreateTemp("", "fab-import-*.tgz")
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Temp fájl hiba: %v", err))
|
|
return
|
|
}
|
|
tmpPath := tmp.Name()
|
|
tmp.Close()
|
|
defer os.Remove(tmpPath)
|
|
|
|
e.debugf("decrypting bundle to %s", tmpPath)
|
|
decStart := time.Now()
|
|
if err := DecryptFile(req.FABPath, tmpPath, req.Password); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Visszafejtés sikertelen: %v", err))
|
|
return
|
|
}
|
|
if decInfo, _ := os.Stat(tmpPath); decInfo != nil {
|
|
e.debugf("decrypted in %v: %s", time.Since(decStart), humanizeBytes(decInfo.Size()))
|
|
}
|
|
tgzPath = tmpPath
|
|
}
|
|
|
|
// Extract tar.gz to temp dir
|
|
tmpDir, err := os.MkdirTemp("", "felhom-import-*")
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Temp könyvtár hiba: %v", err))
|
|
return
|
|
}
|
|
e.debugf("temp dir: %s", tmpDir)
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
e.debugf("extracting tar.gz from %s", tgzPath)
|
|
extractStart := time.Now()
|
|
if err := extractTarGz(tgzPath, tmpDir); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Csomag kicsomagolása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("tar.gz extracted in %v", time.Since(extractStart))
|
|
|
|
// Log extracted contents
|
|
if e.debug {
|
|
filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(tmpDir, path)
|
|
if info.IsDir() {
|
|
e.debugf(" [dir] %s/", rel)
|
|
} else {
|
|
e.debugf(" [file] %s (%s)", rel, humanizeBytes(info.Size()))
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// Read manifest
|
|
manifestData, err := os.ReadFile(filepath.Join(tmpDir, "manifest.json"))
|
|
if err != nil {
|
|
e.failJob(job, step, "manifest.json nem található a csomagban")
|
|
return
|
|
}
|
|
manifest, err := UnmarshalManifest(manifestData)
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Manifest hiba: %v", err))
|
|
return
|
|
}
|
|
|
|
e.debugf("manifest: app=%s display=%s version=%d controller=%s needsHDD=%v",
|
|
manifest.AppName, manifest.DisplayName, manifest.Version, manifest.ControllerVer, manifest.NeedsHDD)
|
|
e.debugf("manifest: hasDB=%v dbType=%s hasHDD=%v hasVolume=%v totalSize=%s",
|
|
manifest.HasDatabase, manifest.DBType, manifest.HasHDDData, manifest.HasVolumeData,
|
|
humanizeBytes(manifest.TotalSizeBytes))
|
|
e.debugf("manifest: configFiles=%v volumes=%v hddSubdirs=%v",
|
|
manifest.ConfigFiles, manifest.VolumeNames, manifest.HDDSubdirs)
|
|
|
|
// Update job with app info
|
|
job.mu.Lock()
|
|
job.StackName = manifest.AppName
|
|
job.DisplayName = manifest.DisplayName
|
|
job.mu.Unlock()
|
|
|
|
e.logger.Printf("[INFO] Import: opening bundle for %s (%s)", manifest.AppName, manifest.DisplayName)
|
|
e.debugf("step 0 (open bundle) done in %v", time.Since(stepStart))
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 1: Prepare app ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
stacksDir := e.provider.GetStacksBaseDir()
|
|
stackDir := filepath.Join(stacksDir, manifest.AppName)
|
|
e.debugf("stacksDir=%s stackDir=%s", stacksDir, stackDir)
|
|
|
|
isDeployed := e.provider.IsStackDeployed(manifest.AppName)
|
|
isRunning := false
|
|
if isDeployed {
|
|
isRunning = e.provider.IsStackRunning(manifest.AppName)
|
|
}
|
|
e.debugf("existing stack: deployed=%v running=%v", isDeployed, isRunning)
|
|
|
|
if isDeployed {
|
|
if isRunning {
|
|
e.logger.Printf("[INFO] Import: removing existing stack %s (with volumes)", manifest.AppName)
|
|
e.debugf("removing existing stack with volumes")
|
|
if err := e.provider.RemoveStackVolumes(manifest.AppName); err != nil {
|
|
e.logger.Printf("[WARN] Import: failed to remove stack volumes: %v", err)
|
|
e.debugf("RemoveStackVolumes failed: %v — trying just StopStack", err)
|
|
_ = e.provider.StopStack(manifest.AppName)
|
|
} else {
|
|
e.debugf("stack volumes removed successfully")
|
|
}
|
|
} else {
|
|
e.debugf("stack is deployed but not running — will overwrite config in place")
|
|
}
|
|
} else {
|
|
e.debugf("creating new stack directory: %s", stackDir)
|
|
if err := os.MkdirAll(stackDir, 0755); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Könyvtár létrehozása sikertelen: %v", err))
|
|
return
|
|
}
|
|
}
|
|
|
|
e.debugf("step 1 (prepare app) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 2: Restore config ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
configDir := filepath.Join(tmpDir, "config")
|
|
e.debugf("restoring config from %s → %s", configDir, stackDir)
|
|
env, err := e.restoreConfig(configDir, stackDir, manifest)
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Konfiguráció visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("config restored: %d env vars", len(env))
|
|
if e.debug {
|
|
for k := range env {
|
|
e.debugf(" env: %s=%s", k, maskSecret(k, env[k]))
|
|
}
|
|
}
|
|
|
|
e.debugf("step 2 (restore config) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 3: Restore user data ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
composePath := filepath.Join(stackDir, "docker-compose.yml")
|
|
|
|
if manifest.HasHDDData {
|
|
e.debugf("restoring HDD data: subdirs=%v", manifest.HDDSubdirs)
|
|
if err := e.restoreHDDData(tmpDir, manifest, composePath, env); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("HDD adatok visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
} else {
|
|
e.debugf("no HDD data in bundle — skipping")
|
|
}
|
|
|
|
if manifest.HasVolumeData {
|
|
e.debugf("restoring Docker volumes: %v", manifest.VolumeNames)
|
|
if err := e.restoreVolumeData(tmpDir, manifest); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Docker volume visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
} else {
|
|
e.debugf("no volume data in bundle — skipping")
|
|
}
|
|
|
|
e.debugf("step 3 (restore user data) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 4: Restore database ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
if manifest.HasDatabase {
|
|
e.debugf("restoring database: type=%s", manifest.DBType)
|
|
if err := e.restoreDatabase(manifest, tmpDir, composePath, stackDir, env); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Adatbázis visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("database restored successfully")
|
|
} else {
|
|
e.debugf("no database in bundle — skipping")
|
|
}
|
|
|
|
e.debugf("step 4 (restore database) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 5: Start app ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
e.debugf("starting stack %s", manifest.AppName)
|
|
if err := e.provider.StartStack(manifest.AppName); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Alkalmazás indítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("stack started in %v", time.Since(stepStart))
|
|
|
|
// Refresh stack list so UI sees the new/updated app
|
|
e.debugf("refreshing stack list")
|
|
if err := e.provider.RefreshStacks(); err != nil {
|
|
e.logger.Printf("[WARN] Import: refresh stacks failed: %v", err)
|
|
e.debugf("RefreshStacks error: %v", err)
|
|
} else {
|
|
e.debugf("stack list refreshed")
|
|
}
|
|
|
|
job.setStep(step, "done", "")
|
|
|
|
e.logger.Printf("[INFO] Import completed: %s (%s) in %v", manifest.AppName, manifest.DisplayName, time.Since(importStart))
|
|
}
|
|
|
|
// maskSecret masks sensitive env var values for debug logging.
|
|
func maskSecret(key, value string) string {
|
|
lower := strings.ToLower(key)
|
|
if strings.Contains(lower, "password") || strings.Contains(lower, "secret") ||
|
|
strings.Contains(lower, "token") || strings.Contains(lower, "key") {
|
|
if len(value) <= 3 {
|
|
return "***"
|
|
}
|
|
return value[:2] + "***"
|
|
}
|
|
return value
|
|
}
|
|
|
|
// restoreConfig extracts config files to the stack directory and re-encrypts app.yaml.
|
|
// Returns the plaintext env map for use in subsequent steps.
|
|
func (e *Exporter) restoreConfig(configDir, stackDir string, manifest *Manifest) (map[string]string, error) {
|
|
entries, err := os.ReadDir(configDir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading config dir: %w", err)
|
|
}
|
|
|
|
e.debugf("restoreConfig: %d entries in config dir", len(entries))
|
|
var env map[string]string
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
|
|
src := filepath.Join(configDir, entry.Name())
|
|
dst := filepath.Join(stackDir, entry.Name())
|
|
|
|
// app.yaml: read plaintext env, then re-encrypt via provider
|
|
if entry.Name() == "app.yaml" {
|
|
e.debugf("restoreConfig: reading plaintext app.yaml from bundle")
|
|
env, err = readPlaintextAppYaml(src)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading app.yaml: %w", err)
|
|
}
|
|
e.debugf("restoreConfig: app.yaml has %d env vars, re-encrypting via provider", len(env))
|
|
if err := e.provider.SaveEncryptedAppConfig(stackDir, env); err != nil {
|
|
return nil, fmt.Errorf("saving encrypted app.yaml: %w", err)
|
|
}
|
|
e.debugf("restoreConfig: app.yaml re-encrypted and saved")
|
|
continue
|
|
}
|
|
|
|
// Copy other config files as-is
|
|
if info, _ := entry.Info(); info != nil {
|
|
e.debugf("restoreConfig: copying %s (%s)", entry.Name(), humanizeBytes(info.Size()))
|
|
}
|
|
if err := copyFile(src, dst); err != nil {
|
|
return nil, fmt.Errorf("copying %s: %w", entry.Name(), err)
|
|
}
|
|
}
|
|
|
|
return env, nil
|
|
}
|
|
|
|
// readPlaintextAppYaml parses the plaintext app.yaml written by the export.
|
|
func readPlaintextAppYaml(path string) (map[string]string, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
env := make(map[string]string)
|
|
inEnv := false
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
|
continue
|
|
}
|
|
|
|
if trimmed == "env:" {
|
|
inEnv = true
|
|
continue
|
|
}
|
|
|
|
if inEnv && strings.HasPrefix(line, " ") {
|
|
parts := strings.SplitN(trimmed, ":", 2)
|
|
if len(parts) == 2 {
|
|
key := strings.TrimSpace(parts[0])
|
|
val := strings.TrimSpace(parts[1])
|
|
// Remove Go-style quotes (written by writeDecryptedAppYaml with %q)
|
|
if len(val) >= 2 && val[0] == '"' && val[len(val)-1] == '"' {
|
|
if unquoted, err := strconv.Unquote(val); err == nil {
|
|
val = unquoted
|
|
}
|
|
}
|
|
env[key] = val
|
|
}
|
|
} else if inEnv && !strings.HasPrefix(line, " ") {
|
|
inEnv = false
|
|
}
|
|
}
|
|
|
|
return env, nil
|
|
}
|
|
|
|
// restoreHDDData extracts HDD bind mount data from the bundle.
|
|
func (e *Exporter) restoreHDDData(tmpDir string, manifest *Manifest, composePath string, env map[string]string) error {
|
|
hddDir := filepath.Join(tmpDir, "data", "hdd")
|
|
|
|
// Resolve HDD mount paths from compose file
|
|
hddPath := env["HDD_PATH"]
|
|
e.debugf("restoreHDDData: HDD_PATH=%s composePath=%s", hddPath, composePath)
|
|
mountByName := make(map[string]string)
|
|
|
|
if hddPath != "" {
|
|
mounts := resolveHDDMounts(composePath, env)
|
|
e.debugf("restoreHDDData: resolved mounts: %v", mounts)
|
|
for _, m := range mounts {
|
|
mountByName[filepath.Base(m)] = m
|
|
}
|
|
} else {
|
|
e.debugf("restoreHDDData: HDD_PATH is empty — will use fallback paths")
|
|
}
|
|
|
|
for _, subdir := range manifest.HDDSubdirs {
|
|
tarPath := filepath.Join(hddDir, subdir+".tar")
|
|
tarInfo, err := os.Stat(tarPath)
|
|
if err != nil {
|
|
e.logger.Printf("[WARN] Import: HDD tar not found: %s", tarPath)
|
|
e.debugf("restoreHDDData: tar not found: %s", tarPath)
|
|
continue
|
|
}
|
|
e.debugf("restoreHDDData: subdir=%s tarSize=%s", subdir, humanizeBytes(tarInfo.Size()))
|
|
|
|
destPath := mountByName[subdir]
|
|
if destPath == "" {
|
|
if filepath.Base(hddPath) == subdir {
|
|
destPath = hddPath
|
|
} else {
|
|
destPath = filepath.Join(hddPath, subdir)
|
|
}
|
|
e.debugf("restoreHDDData: no mount match for %s, using fallback: %s", subdir, destPath)
|
|
} else {
|
|
e.debugf("restoreHDDData: mount match for %s → %s", subdir, destPath)
|
|
}
|
|
|
|
if err := os.MkdirAll(destPath, 0755); err != nil {
|
|
return fmt.Errorf("creating %s: %w", destPath, err)
|
|
}
|
|
|
|
e.logger.Printf("[INFO] Import: extracting HDD data %s → %s", subdir, destPath)
|
|
extractStart := time.Now()
|
|
if err := extractTar(tarPath, destPath); err != nil {
|
|
return fmt.Errorf("extracting %s: %w", subdir, err)
|
|
}
|
|
e.debugf("restoreHDDData: extracted %s in %v", subdir, time.Since(extractStart))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolveHDDMounts parses compose volumes for HDD bind mounts and resolves env vars.
|
|
func resolveHDDMounts(composePath string, env map[string]string) []string {
|
|
data, err := os.ReadFile(composePath)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
hddPath := env["HDD_PATH"]
|
|
if hddPath == "" {
|
|
return nil
|
|
}
|
|
|
|
var mounts []string
|
|
seen := make(map[string]bool)
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "- ") {
|
|
continue
|
|
}
|
|
vol := strings.TrimPrefix(trimmed, "- ")
|
|
// Parse "host:container" or "host:container:mode"
|
|
parts := strings.SplitN(vol, ":", 2)
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
hostPart := parts[0]
|
|
if strings.Contains(hostPart, "${HDD_PATH}") {
|
|
resolved := strings.ReplaceAll(hostPart, "${HDD_PATH}", hddPath)
|
|
resolved = filepath.Clean(resolved)
|
|
if !seen[resolved] {
|
|
seen[resolved] = true
|
|
mounts = append(mounts, resolved)
|
|
}
|
|
}
|
|
}
|
|
|
|
return mounts
|
|
}
|
|
|
|
// restoreVolumeData recreates Docker named volumes from bundle tarballs.
|
|
func (e *Exporter) restoreVolumeData(tmpDir string, manifest *Manifest) error {
|
|
volDir := filepath.Join(tmpDir, "data", "volumes")
|
|
|
|
for _, volName := range manifest.VolumeNames {
|
|
tarPath := filepath.Join(volDir, volName+".tar")
|
|
tarInfo, err := os.Stat(tarPath)
|
|
if err != nil {
|
|
e.logger.Printf("[WARN] Import: volume tar not found: %s", tarPath)
|
|
e.debugf("restoreVolumeData: tar not found: %s", tarPath)
|
|
continue
|
|
}
|
|
e.debugf("restoreVolumeData: volume=%s tarSize=%s", volName, humanizeBytes(tarInfo.Size()))
|
|
|
|
// Create the Docker volume
|
|
e.debugf("restoreVolumeData: creating docker volume %s", volName)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
out, err := exec.CommandContext(ctx, "docker", "volume", "create", volName).CombinedOutput()
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("creating volume %s: %s — %w", volName, strings.TrimSpace(string(out)), err)
|
|
}
|
|
e.debugf("restoreVolumeData: volume %s created: %s", volName, strings.TrimSpace(string(out)))
|
|
|
|
// Populate volume from tar
|
|
e.logger.Printf("[INFO] Import: populating volume %s", volName)
|
|
e.debugf("restoreVolumeData: populating %s via docker run alpine tar xf...", volName)
|
|
popStart := time.Now()
|
|
ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute)
|
|
out, err = exec.CommandContext(ctx, "docker", "run", "--rm",
|
|
"-v", volName+":/vol",
|
|
"-v", volDir+":/in:ro",
|
|
"alpine", "tar", "xf", "/in/"+volName+".tar", "-C", "/vol").CombinedOutput()
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("populating volume %s: %s — %w", volName, strings.TrimSpace(string(out)), err)
|
|
}
|
|
e.debugf("restoreVolumeData: volume %s populated in %v", volName, time.Since(popStart))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// restoreDatabase imports the database dump from the bundle.
|
|
func (e *Exporter) restoreDatabase(manifest *Manifest, tmpDir, composePath, stackDir string, env map[string]string) error {
|
|
dbDir := filepath.Join(tmpDir, "database")
|
|
|
|
// Find the gzipped dump file
|
|
dumpGzPath, err := findDumpFile(dbDir)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: no dump file found in %s: %v", dbDir, err)
|
|
return err
|
|
}
|
|
if dumpInfo, _ := os.Stat(dumpGzPath); dumpInfo != nil {
|
|
e.debugf("restoreDatabase: dump file=%s size=%s", dumpGzPath, humanizeBytes(dumpInfo.Size()))
|
|
}
|
|
|
|
// Find DB service in compose file
|
|
e.debugf("restoreDatabase: parsing compose for DB service: %s", composePath)
|
|
dbSvc := findDBServiceInCompose(composePath)
|
|
if dbSvc == nil {
|
|
e.debugf("restoreDatabase: no DB service found in compose file!")
|
|
return fmt.Errorf("no database service found in docker-compose.yml")
|
|
}
|
|
e.debugf("restoreDatabase: found DB service=%s type=%s container=%s",
|
|
dbSvc.ServiceName, dbSvc.DBType, dbSvc.ContainerName)
|
|
|
|
// Start only the DB service
|
|
e.logger.Printf("[INFO] Import: starting DB service %s (%s)", dbSvc.ServiceName, dbSvc.DBType)
|
|
e.debugf("restoreDatabase: docker compose up -d %s", dbSvc.ServiceName)
|
|
out, err := composeExecEnv(stackDir, env, "up", "-d", dbSvc.ServiceName)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: compose up failed: %s", strings.TrimSpace(string(out)))
|
|
return fmt.Errorf("starting DB service: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: compose up output: %s", strings.TrimSpace(string(out)))
|
|
|
|
// Ensure we stop the DB service after restore (full stack start is in the next step)
|
|
defer func() {
|
|
e.debugf("restoreDatabase: stopping DB service %s", dbSvc.ServiceName)
|
|
out, _ := composeExecEnv(stackDir, env, "stop", dbSvc.ServiceName)
|
|
e.debugf("restoreDatabase: compose stop output: %s", strings.TrimSpace(string(out)))
|
|
}()
|
|
|
|
// Get the container ID
|
|
e.debugf("restoreDatabase: getting container ID for service %s", dbSvc.ServiceName)
|
|
containerID, err := getComposeContainerID(stackDir, env, dbSvc.ServiceName)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: failed to get container ID: %v", err)
|
|
return fmt.Errorf("getting container ID: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: container ID=%s", containerID)
|
|
|
|
// Wait for DB readiness
|
|
e.logger.Printf("[INFO] Import: waiting for DB readiness (container %s)", containerID[:12])
|
|
e.debugf("restoreDatabase: waiting for DB readiness...")
|
|
waitStart := time.Now()
|
|
if err := waitForDB(containerID, dbSvc.DBType, env); err != nil {
|
|
e.debugf("restoreDatabase: DB not ready after %v: %v", time.Since(waitStart), err)
|
|
return fmt.Errorf("waiting for DB: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: DB ready in %v", time.Since(waitStart))
|
|
|
|
// Import the dump
|
|
e.logger.Printf("[INFO] Import: importing database dump")
|
|
e.debugf("restoreDatabase: importing dump into container %s", containerID[:12])
|
|
importStart := time.Now()
|
|
if err := importDBDump(containerID, dumpGzPath, dbSvc.DBType, env); err != nil {
|
|
e.debugf("restoreDatabase: import failed after %v: %v", time.Since(importStart), err)
|
|
return fmt.Errorf("importing dump: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: dump imported in %v", time.Since(importStart))
|
|
|
|
return nil
|
|
}
|
|
|
|
// findDumpFile finds the gzipped SQL dump in the database directory.
|
|
func findDumpFile(dbDir string) (string, error) {
|
|
entries, err := os.ReadDir(dbDir)
|
|
if err != nil {
|
|
return "", fmt.Errorf("reading database dir: %w", err)
|
|
}
|
|
for _, entry := range entries {
|
|
if strings.HasSuffix(entry.Name(), ".sql.gz") || strings.HasSuffix(entry.Name(), ".sql") {
|
|
return filepath.Join(dbDir, entry.Name()), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no database dump file found")
|
|
}
|
|
|
|
// dbServiceInfo describes a database service found in a compose file.
|
|
type dbServiceInfo struct {
|
|
ServiceName string
|
|
DBType string // "postgres" or "mariadb"
|
|
ContainerName string
|
|
}
|
|
|
|
// findDBServiceInCompose parses a compose file to find the database service.
|
|
func findDBServiceInCompose(composePath string) *dbServiceInfo {
|
|
data, err := os.ReadFile(composePath)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
inServices := false
|
|
currentService := ""
|
|
currentImage := ""
|
|
currentContainer := ""
|
|
|
|
checkAndReturn := func() *dbServiceInfo {
|
|
if currentService != "" && currentImage != "" {
|
|
return classifyDBImage(currentService, currentImage, currentContainer)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
|
|
// Top-level section detection
|
|
if len(line) > 0 && line[0] != ' ' && line[0] != '#' {
|
|
if strings.HasPrefix(trimmed, "services:") {
|
|
inServices = true
|
|
} else {
|
|
// Leaving services section — check last service
|
|
if info := checkAndReturn(); info != nil {
|
|
return info
|
|
}
|
|
inServices = false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if !inServices {
|
|
continue
|
|
}
|
|
|
|
// Service name: 2-space indent, ends with colon, no further nesting
|
|
if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") &&
|
|
strings.HasSuffix(trimmed, ":") && !strings.Contains(trimmed, " ") {
|
|
// Check previous service
|
|
if info := checkAndReturn(); info != nil {
|
|
return info
|
|
}
|
|
currentService = strings.TrimSuffix(trimmed, ":")
|
|
currentImage = ""
|
|
currentContainer = ""
|
|
continue
|
|
}
|
|
|
|
// Properties at 4+ space indent
|
|
if strings.HasPrefix(trimmed, "image:") {
|
|
currentImage = strings.TrimSpace(strings.TrimPrefix(trimmed, "image:"))
|
|
}
|
|
if strings.HasPrefix(trimmed, "container_name:") {
|
|
currentContainer = strings.TrimSpace(strings.TrimPrefix(trimmed, "container_name:"))
|
|
}
|
|
}
|
|
|
|
// Check last service
|
|
return checkAndReturn()
|
|
}
|
|
|
|
func classifyDBImage(service, image, container string) *dbServiceInfo {
|
|
img := strings.ToLower(image)
|
|
if strings.Contains(img, "postgres") {
|
|
return &dbServiceInfo{ServiceName: service, DBType: "postgres", ContainerName: container}
|
|
}
|
|
if strings.Contains(img, "mariadb") || strings.Contains(img, "mysql") {
|
|
return &dbServiceInfo{ServiceName: service, DBType: "mariadb", ContainerName: container}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// composeExecEnv runs docker compose in the given stack directory with env vars.
|
|
func composeExecEnv(stackDir string, env map[string]string, args ...string) ([]byte, error) {
|
|
cmdArgs := append([]string{"compose"}, args...)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
cmd := exec.CommandContext(ctx, "docker", cmdArgs...)
|
|
cmd.Dir = stackDir
|
|
cmd.Env = os.Environ()
|
|
for k, v := range env {
|
|
cmd.Env = append(cmd.Env, k+"="+v)
|
|
}
|
|
return cmd.CombinedOutput()
|
|
}
|
|
|
|
// getComposeContainerID returns the container ID for a compose service.
|
|
func getComposeContainerID(stackDir string, env map[string]string, service string) (string, error) {
|
|
// Wait a moment for the container to be created
|
|
time.Sleep(2 * time.Second)
|
|
|
|
out, err := composeExecEnv(stackDir, env, "ps", "-q", service)
|
|
if err != nil {
|
|
return "", fmt.Errorf("compose ps: %w", err)
|
|
}
|
|
id := strings.TrimSpace(string(out))
|
|
if id == "" {
|
|
return "", fmt.Errorf("no container found for service %s", service)
|
|
}
|
|
// Take the first line in case of multiple
|
|
if idx := strings.Index(id, "\n"); idx > 0 {
|
|
id = id[:idx]
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// waitForDB polls until the database is ready to accept connections.
|
|
func waitForDB(containerID, dbType string, env map[string]string) error {
|
|
timeout := 60 * time.Second
|
|
start := time.Now()
|
|
|
|
for {
|
|
if time.Since(start) > timeout {
|
|
return fmt.Errorf("timeout waiting for %s to become ready", dbType)
|
|
}
|
|
|
|
var cmd *exec.Cmd
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
switch dbType {
|
|
case "postgres":
|
|
user := env["POSTGRES_USER"]
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", containerID,
|
|
"pg_isready", "-U", user)
|
|
case "mariadb":
|
|
password := env["MYSQL_ROOT_PASSWORD"]
|
|
if password == "" {
|
|
password = env["MARIADB_ROOT_PASSWORD"]
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", containerID,
|
|
"mysqladmin", "ping", "-u", "root", "-p"+password)
|
|
default:
|
|
cancel()
|
|
return fmt.Errorf("unknown DB type: %s", dbType)
|
|
}
|
|
|
|
err := cmd.Run()
|
|
cancel()
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
}
|
|
|
|
// importDBDump imports a (possibly gzipped) SQL dump into the running database container.
|
|
func importDBDump(containerID, dumpPath, dbType string, env map[string]string) error {
|
|
// Open the dump file
|
|
f, err := os.Open(dumpPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
// If gzipped, wrap in gzip reader
|
|
var reader io.Reader = f
|
|
if strings.HasSuffix(dumpPath, ".gz") {
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return fmt.Errorf("opening gzip: %w", err)
|
|
}
|
|
defer gr.Close()
|
|
reader = gr
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer cancel()
|
|
|
|
var cmd *exec.Cmd
|
|
switch dbType {
|
|
case "postgres":
|
|
user := env["POSTGRES_USER"]
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
dbName := env["POSTGRES_DB"]
|
|
if dbName == "" {
|
|
dbName = user
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", "-i", containerID,
|
|
"psql", "-U", user, "-d", dbName)
|
|
case "mariadb":
|
|
password := env["MYSQL_ROOT_PASSWORD"]
|
|
if password == "" {
|
|
password = env["MARIADB_ROOT_PASSWORD"]
|
|
}
|
|
dbName := env["MYSQL_DATABASE"]
|
|
if dbName == "" {
|
|
dbName = env["MARIADB_DATABASE"]
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", "-i", containerID,
|
|
"mysql", "-u", "root", "-p"+password, dbName)
|
|
default:
|
|
return fmt.Errorf("unknown DB type: %s", dbType)
|
|
}
|
|
|
|
cmd.Stdin = reader
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("%s import failed: %s — %w", dbType, strings.TrimSpace(string(out)), err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// extractTarGz extracts a tar.gz archive to a directory.
|
|
func extractTarGz(tgzPath, destDir string) error {
|
|
f, err := os.Open(tgzPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gr.Close()
|
|
|
|
tr := tar.NewReader(gr)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
target := filepath.Join(destDir, hdr.Name)
|
|
|
|
// Security: prevent path traversal
|
|
if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
|
continue
|
|
}
|
|
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg:
|
|
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
|
return err
|
|
}
|
|
outFile, err := os.Create(target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(outFile, tr); err != nil {
|
|
outFile.Close()
|
|
return err
|
|
}
|
|
outFile.Close()
|
|
os.Chmod(target, os.FileMode(hdr.Mode))
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractTar extracts a plain tar archive to a directory.
|
|
func extractTar(tarPath, destDir string) error {
|
|
f, err := os.Open(tarPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
tr := tar.NewReader(f)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
target := filepath.Join(destDir, hdr.Name)
|
|
|
|
// Security: prevent path traversal
|
|
if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
|
continue
|
|
}
|
|
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg:
|
|
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
|
return err
|
|
}
|
|
outFile, err := os.Create(target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(outFile, tr); err != nil {
|
|
outFile.Close()
|
|
return err
|
|
}
|
|
outFile.Close()
|
|
os.Chmod(target, os.FileMode(hdr.Mode))
|
|
}
|
|
}
|
|
}
|