95c821deb2
Add detailed [DEBUG] logging to every controller module when logging.level is set to "debug". Each module with stateful debug uses SetDebug(bool) wired from main.go. Covers stacks, backup, cloudflare, integrations, system, monitor, settings, scheduler, web handlers, storage, metrics, API, selfupdate, and assets. Also includes the app export/import (.fab bundles) feature from v0.32.0 and its debug page integration. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1120 lines
32 KiB
Go
1120 lines
32 KiB
Go
package appexport
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ScannedBundle describes a .fab file found on a storage drive.
|
|
type ScannedBundle struct {
|
|
Path string `json:"path"`
|
|
FileName string `json:"file_name"`
|
|
AppName string `json:"app_name"`
|
|
DisplayName string `json:"display_name"`
|
|
ExportedAt string `json:"exported_at"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
SizeHuman string `json:"size_human"`
|
|
Encrypted bool `json:"encrypted"`
|
|
NeedsHDD bool `json:"needs_hdd"`
|
|
HasDB bool `json:"has_db"`
|
|
DrivePath string `json:"drive_path"`
|
|
DriveLabel string `json:"drive_label"`
|
|
}
|
|
|
|
// ImportRequest holds user-provided parameters for an import.
|
|
type ImportRequest struct {
|
|
FABPath string // full path to .fab file
|
|
Password string // empty for unencrypted bundles
|
|
}
|
|
|
|
// ScanForBundles scans export directories on all registered drives for .fab files.
|
|
// Pass a non-nil logger to enable debug output.
|
|
func ScanForBundles(drives []DrivePathInfo) []ScannedBundle {
|
|
var bundles []ScannedBundle
|
|
|
|
for _, drive := range drives {
|
|
exportDir := ExportDir(drive.Path)
|
|
entries, err := os.ReadDir(exportDir)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".fab") {
|
|
continue
|
|
}
|
|
|
|
path := filepath.Join(exportDir, entry.Name())
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
bundle := ScannedBundle{
|
|
Path: path,
|
|
FileName: entry.Name(),
|
|
SizeBytes: info.Size(),
|
|
SizeHuman: humanizeBytes(info.Size()),
|
|
DrivePath: drive.Path,
|
|
DriveLabel: drive.Label,
|
|
}
|
|
|
|
// Check encryption
|
|
encrypted, _ := IsEncryptedFAB(path)
|
|
bundle.Encrypted = encrypted
|
|
|
|
// Try to read manifest for metadata (unencrypted only)
|
|
var manifest *Manifest
|
|
if !encrypted {
|
|
manifest, _ = ReadManifestFromFAB(path)
|
|
}
|
|
|
|
if manifest != nil {
|
|
bundle.AppName = manifest.AppName
|
|
bundle.DisplayName = manifest.DisplayName
|
|
bundle.ExportedAt = manifest.ExportedAt.Format("2006-01-02 15:04")
|
|
bundle.NeedsHDD = manifest.NeedsHDD
|
|
bundle.HasDB = manifest.HasDatabase
|
|
} else {
|
|
// Parse app name from filename: {appname}_{timestamp}.fab
|
|
name := strings.TrimSuffix(entry.Name(), ".fab")
|
|
if idx := strings.LastIndex(name, "_"); idx > 0 {
|
|
bundle.AppName = name[:idx]
|
|
} else {
|
|
bundle.AppName = name
|
|
}
|
|
bundle.ExportedAt = info.ModTime().Format("2006-01-02 15:04")
|
|
}
|
|
|
|
bundles = append(bundles, bundle)
|
|
}
|
|
}
|
|
|
|
return bundles
|
|
}
|
|
|
|
// ScanForStaleTempFiles finds stale .fab.tmp and .tgz.tmp files in export directories.
|
|
func ScanForStaleTempFiles(drives []DrivePathInfo) []string {
|
|
var stale []string
|
|
for _, drive := range drives {
|
|
exportDir := ExportDir(drive.Path)
|
|
entries, err := os.ReadDir(exportDir)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
if strings.HasSuffix(name, ".tmp") || strings.HasSuffix(name, ".tgz.tmp") {
|
|
stale = append(stale, filepath.Join(exportDir, name))
|
|
}
|
|
}
|
|
}
|
|
return stale
|
|
}
|
|
|
|
// ReadManifestFromFAB reads the manifest from an unencrypted .fab file.
|
|
func ReadManifestFromFAB(fabPath string) (*Manifest, error) {
|
|
f, err := os.Open(fabPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("not a valid gzip: %w", err)
|
|
}
|
|
defer gr.Close()
|
|
|
|
tr := tar.NewReader(gr)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil, fmt.Errorf("manifest.json not found in bundle")
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if hdr.Name == "manifest.json" {
|
|
data, err := io.ReadAll(tr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return UnmarshalManifest(data)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ReadManifestFromEncryptedFAB decrypts and reads the manifest from an encrypted .fab file.
|
|
func ReadManifestFromEncryptedFAB(fabPath, password string) (*Manifest, error) {
|
|
tmpFile, err := os.CreateTemp("", "fab-manifest-*.tgz")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
tmpFile.Close()
|
|
defer os.Remove(tmpPath)
|
|
|
|
if err := DecryptFile(fabPath, tmpPath, password); err != nil {
|
|
return nil, fmt.Errorf("visszafejtés sikertelen: %w", err)
|
|
}
|
|
|
|
return ReadManifestFromFAB(tmpPath)
|
|
}
|
|
|
|
// StartImport validates and starts an async import. Returns error if blocked.
|
|
func (e *Exporter) StartImport(req ImportRequest) error {
|
|
e.mu.Lock()
|
|
if e.activeJob != nil && e.activeJob.Running {
|
|
e.mu.Unlock()
|
|
e.debugf("StartImport rejected: another job is already running")
|
|
return fmt.Errorf("export or import already in progress")
|
|
}
|
|
|
|
// Validate file exists
|
|
info, err := os.Stat(req.FABPath)
|
|
if err != nil {
|
|
e.mu.Unlock()
|
|
e.debugf("StartImport rejected: file not found: %s", req.FABPath)
|
|
return fmt.Errorf("bundle file not found: %w", err)
|
|
}
|
|
|
|
e.debugf("StartImport: path=%s size=%s encrypted=%v",
|
|
req.FABPath, humanizeBytes(info.Size()), req.Password != "")
|
|
|
|
steps := []Step{
|
|
{Label: "Csomag megnyitása", Status: "pending"},
|
|
{Label: "Alkalmazás előkészítése", Status: "pending"},
|
|
{Label: "Konfiguráció visszaállítása", Status: "pending"},
|
|
{Label: "Felhasználói adatok visszaállítása", Status: "pending"},
|
|
{Label: "Adatbázis visszaállítása", Status: "pending"},
|
|
{Label: "Alkalmazás indítása", Status: "pending"},
|
|
}
|
|
|
|
job := &Job{
|
|
StackName: filepath.Base(req.FABPath),
|
|
Steps: steps,
|
|
Running: true,
|
|
JobType: "import",
|
|
}
|
|
e.activeJob = job
|
|
e.mu.Unlock()
|
|
|
|
go e.executeImport(req, job)
|
|
return nil
|
|
}
|
|
|
|
func (e *Exporter) executeImport(req ImportRequest, job *Job) {
|
|
importStart := time.Now()
|
|
e.debugf("=== IMPORT START: path=%s encrypted=%v ===", req.FABPath, req.Password != "")
|
|
|
|
defer func() {
|
|
job.mu.Lock()
|
|
job.Running = false
|
|
job.Done = true
|
|
job.mu.Unlock()
|
|
e.debugf("=== IMPORT END: elapsed=%v ===", time.Since(importStart))
|
|
}()
|
|
|
|
step := 0
|
|
|
|
// --- Step 0: Open bundle ---
|
|
job.setStep(step, "running", "")
|
|
stepStart := time.Now()
|
|
|
|
encrypted, _ := IsEncryptedFAB(req.FABPath)
|
|
e.debugf("bundle encrypted: %v", encrypted)
|
|
tgzPath := req.FABPath
|
|
|
|
// Decrypt if needed
|
|
if encrypted {
|
|
if req.Password == "" {
|
|
e.failJob(job, step, "Titkosított csomag — jelszó szükséges")
|
|
return
|
|
}
|
|
tmp, err := os.CreateTemp("", "fab-import-*.tgz")
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Temp fájl hiba: %v", err))
|
|
return
|
|
}
|
|
tmpPath := tmp.Name()
|
|
tmp.Close()
|
|
defer os.Remove(tmpPath)
|
|
|
|
e.debugf("decrypting bundle to %s", tmpPath)
|
|
decStart := time.Now()
|
|
if err := DecryptFile(req.FABPath, tmpPath, req.Password); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Visszafejtés sikertelen: %v", err))
|
|
return
|
|
}
|
|
if decInfo, _ := os.Stat(tmpPath); decInfo != nil {
|
|
e.debugf("decrypted in %v: %s", time.Since(decStart), humanizeBytes(decInfo.Size()))
|
|
}
|
|
tgzPath = tmpPath
|
|
}
|
|
|
|
// Extract tar.gz to temp dir
|
|
tmpDir, err := os.MkdirTemp("", "felhom-import-*")
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Temp könyvtár hiba: %v", err))
|
|
return
|
|
}
|
|
e.debugf("temp dir: %s", tmpDir)
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
e.debugf("extracting tar.gz from %s", tgzPath)
|
|
extractStart := time.Now()
|
|
if err := extractTarGz(tgzPath, tmpDir); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Csomag kicsomagolása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("tar.gz extracted in %v", time.Since(extractStart))
|
|
|
|
// Log extracted contents
|
|
if e.debug {
|
|
filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(tmpDir, path)
|
|
if info.IsDir() {
|
|
e.debugf(" [dir] %s/", rel)
|
|
} else {
|
|
e.debugf(" [file] %s (%s)", rel, humanizeBytes(info.Size()))
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// Read manifest
|
|
manifestData, err := os.ReadFile(filepath.Join(tmpDir, "manifest.json"))
|
|
if err != nil {
|
|
e.failJob(job, step, "manifest.json nem található a csomagban")
|
|
return
|
|
}
|
|
manifest, err := UnmarshalManifest(manifestData)
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Manifest hiba: %v", err))
|
|
return
|
|
}
|
|
|
|
e.debugf("manifest: app=%s display=%s version=%d controller=%s needsHDD=%v",
|
|
manifest.AppName, manifest.DisplayName, manifest.Version, manifest.ControllerVer, manifest.NeedsHDD)
|
|
e.debugf("manifest: hasDB=%v dbType=%s hasHDD=%v hasVolume=%v totalSize=%s",
|
|
manifest.HasDatabase, manifest.DBType, manifest.HasHDDData, manifest.HasVolumeData,
|
|
humanizeBytes(manifest.TotalSizeBytes))
|
|
e.debugf("manifest: configFiles=%v volumes=%v hddSubdirs=%v",
|
|
manifest.ConfigFiles, manifest.VolumeNames, manifest.HDDSubdirs)
|
|
|
|
// Update job with app info
|
|
job.mu.Lock()
|
|
job.StackName = manifest.AppName
|
|
job.DisplayName = manifest.DisplayName
|
|
job.mu.Unlock()
|
|
|
|
e.logger.Printf("[INFO] Import: opening bundle for %s (%s)", manifest.AppName, manifest.DisplayName)
|
|
e.debugf("step 0 (open bundle) done in %v", time.Since(stepStart))
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 1: Prepare app ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
stacksDir := e.provider.GetStacksBaseDir()
|
|
stackDir := filepath.Join(stacksDir, manifest.AppName)
|
|
e.debugf("stacksDir=%s stackDir=%s", stacksDir, stackDir)
|
|
|
|
isDeployed := e.provider.IsStackDeployed(manifest.AppName)
|
|
isRunning := false
|
|
if isDeployed {
|
|
isRunning = e.provider.IsStackRunning(manifest.AppName)
|
|
}
|
|
e.debugf("existing stack: deployed=%v running=%v", isDeployed, isRunning)
|
|
|
|
if isDeployed {
|
|
if isRunning {
|
|
e.logger.Printf("[INFO] Import: removing existing stack %s (with volumes)", manifest.AppName)
|
|
e.debugf("removing existing stack with volumes")
|
|
if err := e.provider.RemoveStackVolumes(manifest.AppName); err != nil {
|
|
e.logger.Printf("[WARN] Import: failed to remove stack volumes: %v", err)
|
|
e.debugf("RemoveStackVolumes failed: %v — trying just StopStack", err)
|
|
_ = e.provider.StopStack(manifest.AppName)
|
|
} else {
|
|
e.debugf("stack volumes removed successfully")
|
|
}
|
|
} else {
|
|
e.debugf("stack is deployed but not running — will overwrite config in place")
|
|
}
|
|
} else {
|
|
e.debugf("creating new stack directory: %s", stackDir)
|
|
if err := os.MkdirAll(stackDir, 0755); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Könyvtár létrehozása sikertelen: %v", err))
|
|
return
|
|
}
|
|
}
|
|
|
|
e.debugf("step 1 (prepare app) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 2: Restore config ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
configDir := filepath.Join(tmpDir, "config")
|
|
e.debugf("restoring config from %s → %s", configDir, stackDir)
|
|
env, err := e.restoreConfig(configDir, stackDir, manifest)
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Konfiguráció visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("config restored: %d env vars", len(env))
|
|
if e.debug {
|
|
for k := range env {
|
|
e.debugf(" env: %s=%s", k, maskSecret(k, env[k]))
|
|
}
|
|
}
|
|
|
|
e.debugf("step 2 (restore config) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 3: Restore user data ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
composePath := filepath.Join(stackDir, "docker-compose.yml")
|
|
|
|
if manifest.HasHDDData {
|
|
e.debugf("restoring HDD data: subdirs=%v", manifest.HDDSubdirs)
|
|
if err := e.restoreHDDData(tmpDir, manifest, composePath, env); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("HDD adatok visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
} else {
|
|
e.debugf("no HDD data in bundle — skipping")
|
|
}
|
|
|
|
if manifest.HasVolumeData {
|
|
e.debugf("restoring Docker volumes: %v", manifest.VolumeNames)
|
|
if err := e.restoreVolumeData(tmpDir, manifest); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Docker volume visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
} else {
|
|
e.debugf("no volume data in bundle — skipping")
|
|
}
|
|
|
|
e.debugf("step 3 (restore user data) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 4: Restore database ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
if manifest.HasDatabase {
|
|
e.debugf("restoring database: type=%s", manifest.DBType)
|
|
if err := e.restoreDatabase(manifest, tmpDir, composePath, stackDir, env); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Adatbázis visszaállítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("database restored successfully")
|
|
} else {
|
|
e.debugf("no database in bundle — skipping")
|
|
}
|
|
|
|
e.debugf("step 4 (restore database) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 5: Start app ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
e.debugf("starting stack %s", manifest.AppName)
|
|
if err := e.provider.StartStack(manifest.AppName); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Alkalmazás indítása sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("stack started in %v", time.Since(stepStart))
|
|
|
|
// Refresh stack list so UI sees the new/updated app
|
|
e.debugf("refreshing stack list")
|
|
if err := e.provider.RefreshStacks(); err != nil {
|
|
e.logger.Printf("[WARN] Import: refresh stacks failed: %v", err)
|
|
e.debugf("RefreshStacks error: %v", err)
|
|
} else {
|
|
e.debugf("stack list refreshed")
|
|
}
|
|
|
|
job.setStep(step, "done", "")
|
|
|
|
e.logger.Printf("[INFO] Import completed: %s (%s) in %v", manifest.AppName, manifest.DisplayName, time.Since(importStart))
|
|
}
|
|
|
|
// maskSecret masks sensitive env var values for debug logging.
|
|
func maskSecret(key, value string) string {
|
|
lower := strings.ToLower(key)
|
|
if strings.Contains(lower, "password") || strings.Contains(lower, "secret") ||
|
|
strings.Contains(lower, "token") || strings.Contains(lower, "key") {
|
|
if len(value) <= 3 {
|
|
return "***"
|
|
}
|
|
return value[:2] + "***"
|
|
}
|
|
return value
|
|
}
|
|
|
|
// restoreConfig extracts config files to the stack directory and re-encrypts app.yaml.
|
|
// Returns the plaintext env map for use in subsequent steps.
|
|
func (e *Exporter) restoreConfig(configDir, stackDir string, manifest *Manifest) (map[string]string, error) {
|
|
entries, err := os.ReadDir(configDir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading config dir: %w", err)
|
|
}
|
|
|
|
e.debugf("restoreConfig: %d entries in config dir", len(entries))
|
|
var env map[string]string
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
|
|
src := filepath.Join(configDir, entry.Name())
|
|
dst := filepath.Join(stackDir, entry.Name())
|
|
|
|
// app.yaml: read plaintext env, then re-encrypt via provider
|
|
if entry.Name() == "app.yaml" {
|
|
e.debugf("restoreConfig: reading plaintext app.yaml from bundle")
|
|
env, err = readPlaintextAppYaml(src)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading app.yaml: %w", err)
|
|
}
|
|
e.debugf("restoreConfig: app.yaml has %d env vars, re-encrypting via provider", len(env))
|
|
if err := e.provider.SaveEncryptedAppConfig(stackDir, env); err != nil {
|
|
return nil, fmt.Errorf("saving encrypted app.yaml: %w", err)
|
|
}
|
|
e.debugf("restoreConfig: app.yaml re-encrypted and saved")
|
|
continue
|
|
}
|
|
|
|
// Copy other config files as-is
|
|
if info, _ := entry.Info(); info != nil {
|
|
e.debugf("restoreConfig: copying %s (%s)", entry.Name(), humanizeBytes(info.Size()))
|
|
}
|
|
if err := copyFile(src, dst); err != nil {
|
|
return nil, fmt.Errorf("copying %s: %w", entry.Name(), err)
|
|
}
|
|
}
|
|
|
|
return env, nil
|
|
}
|
|
|
|
// readPlaintextAppYaml parses the plaintext app.yaml written by the export.
|
|
func readPlaintextAppYaml(path string) (map[string]string, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
env := make(map[string]string)
|
|
inEnv := false
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
|
continue
|
|
}
|
|
|
|
if trimmed == "env:" {
|
|
inEnv = true
|
|
continue
|
|
}
|
|
|
|
if inEnv && strings.HasPrefix(line, " ") {
|
|
parts := strings.SplitN(trimmed, ":", 2)
|
|
if len(parts) == 2 {
|
|
key := strings.TrimSpace(parts[0])
|
|
val := strings.TrimSpace(parts[1])
|
|
// Remove Go-style quotes (written by writeDecryptedAppYaml with %q)
|
|
if len(val) >= 2 && val[0] == '"' && val[len(val)-1] == '"' {
|
|
if unquoted, err := strconv.Unquote(val); err == nil {
|
|
val = unquoted
|
|
}
|
|
}
|
|
env[key] = val
|
|
}
|
|
} else if inEnv && !strings.HasPrefix(line, " ") {
|
|
inEnv = false
|
|
}
|
|
}
|
|
|
|
return env, nil
|
|
}
|
|
|
|
// restoreHDDData extracts HDD bind mount data from the bundle.
|
|
func (e *Exporter) restoreHDDData(tmpDir string, manifest *Manifest, composePath string, env map[string]string) error {
|
|
hddDir := filepath.Join(tmpDir, "data", "hdd")
|
|
|
|
// Resolve HDD mount paths from compose file
|
|
hddPath := env["HDD_PATH"]
|
|
e.debugf("restoreHDDData: HDD_PATH=%s composePath=%s", hddPath, composePath)
|
|
mountByName := make(map[string]string)
|
|
|
|
if hddPath != "" {
|
|
mounts := resolveHDDMounts(composePath, env)
|
|
e.debugf("restoreHDDData: resolved mounts: %v", mounts)
|
|
for _, m := range mounts {
|
|
mountByName[filepath.Base(m)] = m
|
|
}
|
|
} else {
|
|
e.debugf("restoreHDDData: HDD_PATH is empty — will use fallback paths")
|
|
}
|
|
|
|
for _, subdir := range manifest.HDDSubdirs {
|
|
tarPath := filepath.Join(hddDir, subdir+".tar")
|
|
tarInfo, err := os.Stat(tarPath)
|
|
if err != nil {
|
|
e.logger.Printf("[WARN] Import: HDD tar not found: %s", tarPath)
|
|
e.debugf("restoreHDDData: tar not found: %s", tarPath)
|
|
continue
|
|
}
|
|
e.debugf("restoreHDDData: subdir=%s tarSize=%s", subdir, humanizeBytes(tarInfo.Size()))
|
|
|
|
destPath := mountByName[subdir]
|
|
if destPath == "" {
|
|
if filepath.Base(hddPath) == subdir {
|
|
destPath = hddPath
|
|
} else {
|
|
destPath = filepath.Join(hddPath, subdir)
|
|
}
|
|
e.debugf("restoreHDDData: no mount match for %s, using fallback: %s", subdir, destPath)
|
|
} else {
|
|
e.debugf("restoreHDDData: mount match for %s → %s", subdir, destPath)
|
|
}
|
|
|
|
if err := os.MkdirAll(destPath, 0755); err != nil {
|
|
return fmt.Errorf("creating %s: %w", destPath, err)
|
|
}
|
|
|
|
e.logger.Printf("[INFO] Import: extracting HDD data %s → %s", subdir, destPath)
|
|
extractStart := time.Now()
|
|
if err := extractTar(tarPath, destPath); err != nil {
|
|
return fmt.Errorf("extracting %s: %w", subdir, err)
|
|
}
|
|
e.debugf("restoreHDDData: extracted %s in %v", subdir, time.Since(extractStart))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolveHDDMounts parses compose volumes for HDD bind mounts and resolves env vars.
|
|
func resolveHDDMounts(composePath string, env map[string]string) []string {
|
|
data, err := os.ReadFile(composePath)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
hddPath := env["HDD_PATH"]
|
|
if hddPath == "" {
|
|
return nil
|
|
}
|
|
|
|
var mounts []string
|
|
seen := make(map[string]bool)
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "- ") {
|
|
continue
|
|
}
|
|
vol := strings.TrimPrefix(trimmed, "- ")
|
|
// Parse "host:container" or "host:container:mode"
|
|
parts := strings.SplitN(vol, ":", 2)
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
hostPart := parts[0]
|
|
if strings.Contains(hostPart, "${HDD_PATH}") {
|
|
resolved := strings.ReplaceAll(hostPart, "${HDD_PATH}", hddPath)
|
|
resolved = filepath.Clean(resolved)
|
|
if !seen[resolved] {
|
|
seen[resolved] = true
|
|
mounts = append(mounts, resolved)
|
|
}
|
|
}
|
|
}
|
|
|
|
return mounts
|
|
}
|
|
|
|
// restoreVolumeData recreates Docker named volumes from bundle tarballs.
|
|
func (e *Exporter) restoreVolumeData(tmpDir string, manifest *Manifest) error {
|
|
volDir := filepath.Join(tmpDir, "data", "volumes")
|
|
|
|
for _, volName := range manifest.VolumeNames {
|
|
tarPath := filepath.Join(volDir, volName+".tar")
|
|
tarInfo, err := os.Stat(tarPath)
|
|
if err != nil {
|
|
e.logger.Printf("[WARN] Import: volume tar not found: %s", tarPath)
|
|
e.debugf("restoreVolumeData: tar not found: %s", tarPath)
|
|
continue
|
|
}
|
|
e.debugf("restoreVolumeData: volume=%s tarSize=%s", volName, humanizeBytes(tarInfo.Size()))
|
|
|
|
// Create the Docker volume
|
|
e.debugf("restoreVolumeData: creating docker volume %s", volName)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
out, err := exec.CommandContext(ctx, "docker", "volume", "create", volName).CombinedOutput()
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("creating volume %s: %s — %w", volName, strings.TrimSpace(string(out)), err)
|
|
}
|
|
e.debugf("restoreVolumeData: volume %s created: %s", volName, strings.TrimSpace(string(out)))
|
|
|
|
// Populate volume from tar
|
|
e.logger.Printf("[INFO] Import: populating volume %s", volName)
|
|
e.debugf("restoreVolumeData: populating %s via docker run alpine tar xf...", volName)
|
|
popStart := time.Now()
|
|
ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute)
|
|
out, err = exec.CommandContext(ctx, "docker", "run", "--rm",
|
|
"-v", volName+":/vol",
|
|
"-v", volDir+":/in:ro",
|
|
"alpine", "tar", "xf", "/in/"+volName+".tar", "-C", "/vol").CombinedOutput()
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("populating volume %s: %s — %w", volName, strings.TrimSpace(string(out)), err)
|
|
}
|
|
e.debugf("restoreVolumeData: volume %s populated in %v", volName, time.Since(popStart))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// restoreDatabase imports the database dump from the bundle.
|
|
func (e *Exporter) restoreDatabase(manifest *Manifest, tmpDir, composePath, stackDir string, env map[string]string) error {
|
|
dbDir := filepath.Join(tmpDir, "database")
|
|
|
|
// Find the gzipped dump file
|
|
dumpGzPath, err := findDumpFile(dbDir)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: no dump file found in %s: %v", dbDir, err)
|
|
return err
|
|
}
|
|
if dumpInfo, _ := os.Stat(dumpGzPath); dumpInfo != nil {
|
|
e.debugf("restoreDatabase: dump file=%s size=%s", dumpGzPath, humanizeBytes(dumpInfo.Size()))
|
|
}
|
|
|
|
// Find DB service in compose file
|
|
e.debugf("restoreDatabase: parsing compose for DB service: %s", composePath)
|
|
dbSvc := findDBServiceInCompose(composePath)
|
|
if dbSvc == nil {
|
|
e.debugf("restoreDatabase: no DB service found in compose file!")
|
|
return fmt.Errorf("no database service found in docker-compose.yml")
|
|
}
|
|
e.debugf("restoreDatabase: found DB service=%s type=%s container=%s",
|
|
dbSvc.ServiceName, dbSvc.DBType, dbSvc.ContainerName)
|
|
|
|
// Start only the DB service
|
|
e.logger.Printf("[INFO] Import: starting DB service %s (%s)", dbSvc.ServiceName, dbSvc.DBType)
|
|
e.debugf("restoreDatabase: docker compose up -d %s", dbSvc.ServiceName)
|
|
out, err := composeExecEnv(stackDir, env, "up", "-d", dbSvc.ServiceName)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: compose up failed: %s", strings.TrimSpace(string(out)))
|
|
return fmt.Errorf("starting DB service: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: compose up output: %s", strings.TrimSpace(string(out)))
|
|
|
|
// Ensure we stop the DB service after restore (full stack start is in the next step)
|
|
defer func() {
|
|
e.debugf("restoreDatabase: stopping DB service %s", dbSvc.ServiceName)
|
|
out, _ := composeExecEnv(stackDir, env, "stop", dbSvc.ServiceName)
|
|
e.debugf("restoreDatabase: compose stop output: %s", strings.TrimSpace(string(out)))
|
|
}()
|
|
|
|
// Get the container ID
|
|
e.debugf("restoreDatabase: getting container ID for service %s", dbSvc.ServiceName)
|
|
containerID, err := getComposeContainerID(stackDir, env, dbSvc.ServiceName)
|
|
if err != nil {
|
|
e.debugf("restoreDatabase: failed to get container ID: %v", err)
|
|
return fmt.Errorf("getting container ID: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: container ID=%s", containerID)
|
|
|
|
// Wait for DB readiness
|
|
e.logger.Printf("[INFO] Import: waiting for DB readiness (container %s)", containerID[:12])
|
|
e.debugf("restoreDatabase: waiting for DB readiness...")
|
|
waitStart := time.Now()
|
|
if err := waitForDB(containerID, dbSvc.DBType, env); err != nil {
|
|
e.debugf("restoreDatabase: DB not ready after %v: %v", time.Since(waitStart), err)
|
|
return fmt.Errorf("waiting for DB: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: DB ready in %v", time.Since(waitStart))
|
|
|
|
// Import the dump
|
|
e.logger.Printf("[INFO] Import: importing database dump")
|
|
e.debugf("restoreDatabase: importing dump into container %s", containerID[:12])
|
|
importStart := time.Now()
|
|
if err := importDBDump(containerID, dumpGzPath, dbSvc.DBType, env); err != nil {
|
|
e.debugf("restoreDatabase: import failed after %v: %v", time.Since(importStart), err)
|
|
return fmt.Errorf("importing dump: %w", err)
|
|
}
|
|
e.debugf("restoreDatabase: dump imported in %v", time.Since(importStart))
|
|
|
|
return nil
|
|
}
|
|
|
|
// findDumpFile finds the gzipped SQL dump in the database directory.
|
|
func findDumpFile(dbDir string) (string, error) {
|
|
entries, err := os.ReadDir(dbDir)
|
|
if err != nil {
|
|
return "", fmt.Errorf("reading database dir: %w", err)
|
|
}
|
|
for _, entry := range entries {
|
|
if strings.HasSuffix(entry.Name(), ".sql.gz") || strings.HasSuffix(entry.Name(), ".sql") {
|
|
return filepath.Join(dbDir, entry.Name()), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no database dump file found")
|
|
}
|
|
|
|
// dbServiceInfo describes a database service found in a compose file.
|
|
type dbServiceInfo struct {
|
|
ServiceName string
|
|
DBType string // "postgres" or "mariadb"
|
|
ContainerName string
|
|
}
|
|
|
|
// findDBServiceInCompose parses a compose file to find the database service.
|
|
func findDBServiceInCompose(composePath string) *dbServiceInfo {
|
|
data, err := os.ReadFile(composePath)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
inServices := false
|
|
currentService := ""
|
|
currentImage := ""
|
|
currentContainer := ""
|
|
|
|
checkAndReturn := func() *dbServiceInfo {
|
|
if currentService != "" && currentImage != "" {
|
|
return classifyDBImage(currentService, currentImage, currentContainer)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
|
|
// Top-level section detection
|
|
if len(line) > 0 && line[0] != ' ' && line[0] != '#' {
|
|
if strings.HasPrefix(trimmed, "services:") {
|
|
inServices = true
|
|
} else {
|
|
// Leaving services section — check last service
|
|
if info := checkAndReturn(); info != nil {
|
|
return info
|
|
}
|
|
inServices = false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if !inServices {
|
|
continue
|
|
}
|
|
|
|
// Service name: 2-space indent, ends with colon, no further nesting
|
|
if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") &&
|
|
strings.HasSuffix(trimmed, ":") && !strings.Contains(trimmed, " ") {
|
|
// Check previous service
|
|
if info := checkAndReturn(); info != nil {
|
|
return info
|
|
}
|
|
currentService = strings.TrimSuffix(trimmed, ":")
|
|
currentImage = ""
|
|
currentContainer = ""
|
|
continue
|
|
}
|
|
|
|
// Properties at 4+ space indent
|
|
if strings.HasPrefix(trimmed, "image:") {
|
|
currentImage = strings.TrimSpace(strings.TrimPrefix(trimmed, "image:"))
|
|
}
|
|
if strings.HasPrefix(trimmed, "container_name:") {
|
|
currentContainer = strings.TrimSpace(strings.TrimPrefix(trimmed, "container_name:"))
|
|
}
|
|
}
|
|
|
|
// Check last service
|
|
return checkAndReturn()
|
|
}
|
|
|
|
func classifyDBImage(service, image, container string) *dbServiceInfo {
|
|
img := strings.ToLower(image)
|
|
if strings.Contains(img, "postgres") {
|
|
return &dbServiceInfo{ServiceName: service, DBType: "postgres", ContainerName: container}
|
|
}
|
|
if strings.Contains(img, "mariadb") || strings.Contains(img, "mysql") {
|
|
return &dbServiceInfo{ServiceName: service, DBType: "mariadb", ContainerName: container}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// composeExecEnv runs docker compose in the given stack directory with env vars.
|
|
func composeExecEnv(stackDir string, env map[string]string, args ...string) ([]byte, error) {
|
|
cmdArgs := append([]string{"compose"}, args...)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
cmd := exec.CommandContext(ctx, "docker", cmdArgs...)
|
|
cmd.Dir = stackDir
|
|
cmd.Env = os.Environ()
|
|
for k, v := range env {
|
|
cmd.Env = append(cmd.Env, k+"="+v)
|
|
}
|
|
return cmd.CombinedOutput()
|
|
}
|
|
|
|
// getComposeContainerID returns the container ID for a compose service.
|
|
func getComposeContainerID(stackDir string, env map[string]string, service string) (string, error) {
|
|
// Wait a moment for the container to be created
|
|
time.Sleep(2 * time.Second)
|
|
|
|
out, err := composeExecEnv(stackDir, env, "ps", "-q", service)
|
|
if err != nil {
|
|
return "", fmt.Errorf("compose ps: %w", err)
|
|
}
|
|
id := strings.TrimSpace(string(out))
|
|
if id == "" {
|
|
return "", fmt.Errorf("no container found for service %s", service)
|
|
}
|
|
// Take the first line in case of multiple
|
|
if idx := strings.Index(id, "\n"); idx > 0 {
|
|
id = id[:idx]
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// waitForDB polls until the database is ready to accept connections.
|
|
func waitForDB(containerID, dbType string, env map[string]string) error {
|
|
timeout := 60 * time.Second
|
|
start := time.Now()
|
|
|
|
for {
|
|
if time.Since(start) > timeout {
|
|
return fmt.Errorf("timeout waiting for %s to become ready", dbType)
|
|
}
|
|
|
|
var cmd *exec.Cmd
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
switch dbType {
|
|
case "postgres":
|
|
user := env["POSTGRES_USER"]
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", containerID,
|
|
"pg_isready", "-U", user)
|
|
case "mariadb":
|
|
password := env["MYSQL_ROOT_PASSWORD"]
|
|
if password == "" {
|
|
password = env["MARIADB_ROOT_PASSWORD"]
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", containerID,
|
|
"mysqladmin", "ping", "-u", "root", "-p"+password)
|
|
default:
|
|
cancel()
|
|
return fmt.Errorf("unknown DB type: %s", dbType)
|
|
}
|
|
|
|
err := cmd.Run()
|
|
cancel()
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
}
|
|
|
|
// importDBDump imports a (possibly gzipped) SQL dump into the running database container.
|
|
func importDBDump(containerID, dumpPath, dbType string, env map[string]string) error {
|
|
// Open the dump file
|
|
f, err := os.Open(dumpPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
// If gzipped, wrap in gzip reader
|
|
var reader io.Reader = f
|
|
if strings.HasSuffix(dumpPath, ".gz") {
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return fmt.Errorf("opening gzip: %w", err)
|
|
}
|
|
defer gr.Close()
|
|
reader = gr
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer cancel()
|
|
|
|
var cmd *exec.Cmd
|
|
switch dbType {
|
|
case "postgres":
|
|
user := env["POSTGRES_USER"]
|
|
if user == "" {
|
|
user = "postgres"
|
|
}
|
|
dbName := env["POSTGRES_DB"]
|
|
if dbName == "" {
|
|
dbName = user
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", "-i", containerID,
|
|
"psql", "-U", user, "-d", dbName)
|
|
case "mariadb":
|
|
password := env["MYSQL_ROOT_PASSWORD"]
|
|
if password == "" {
|
|
password = env["MARIADB_ROOT_PASSWORD"]
|
|
}
|
|
dbName := env["MYSQL_DATABASE"]
|
|
if dbName == "" {
|
|
dbName = env["MARIADB_DATABASE"]
|
|
}
|
|
cmd = exec.CommandContext(ctx, "docker", "exec", "-i", containerID,
|
|
"mysql", "-u", "root", "-p"+password, dbName)
|
|
default:
|
|
return fmt.Errorf("unknown DB type: %s", dbType)
|
|
}
|
|
|
|
cmd.Stdin = reader
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return fmt.Errorf("%s import failed: %s — %w", dbType, strings.TrimSpace(string(out)), err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// extractTarGz extracts a tar.gz archive to a directory.
|
|
func extractTarGz(tgzPath, destDir string) error {
|
|
f, err := os.Open(tgzPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
gr, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gr.Close()
|
|
|
|
tr := tar.NewReader(gr)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
target := filepath.Join(destDir, hdr.Name)
|
|
|
|
// Security: prevent path traversal
|
|
if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
|
continue
|
|
}
|
|
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg:
|
|
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
|
return err
|
|
}
|
|
outFile, err := os.Create(target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(outFile, tr); err != nil {
|
|
outFile.Close()
|
|
return err
|
|
}
|
|
outFile.Close()
|
|
os.Chmod(target, os.FileMode(hdr.Mode))
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractTar extracts a plain tar archive to a directory.
|
|
func extractTar(tarPath, destDir string) error {
|
|
f, err := os.Open(tarPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
tr := tar.NewReader(f)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
target := filepath.Join(destDir, hdr.Name)
|
|
|
|
// Security: prevent path traversal
|
|
if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
|
continue
|
|
}
|
|
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg:
|
|
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
|
return err
|
|
}
|
|
outFile, err := os.Create(target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(outFile, tr); err != nil {
|
|
outFile.Close()
|
|
return err
|
|
}
|
|
outFile.Close()
|
|
os.Chmod(target, os.FileMode(hdr.Mode))
|
|
}
|
|
}
|
|
}
|