Files
felhom-controller/controller/internal/appexport/restore.go
T
admin a3ed63ee9a
gates / gates (push) Successful in 50s
fix(R-1187): the import's compose runs get the controller's path variables; a compose warning is never read as the container id
Found live on 9202 (grimmory import timed out on the DB step and left the app down). Red-proved.
2026-10-26 21:38:47 +01:00

1375 lines
45 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package appexport
import (
"archive/tar"
"compress/gzip"
"context"
"fmt"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
// ScannedBundle describes a .fab file found on a storage drive.
type ScannedBundle struct {
Path string `json:"path"`
FileName string `json:"file_name"`
AppName string `json:"app_name"`
DisplayName string `json:"display_name"`
ExportedAt string `json:"exported_at"`
SizeBytes int64 `json:"size_bytes"`
SizeHuman string `json:"size_human"`
Encrypted bool `json:"encrypted"`
NeedsHDD bool `json:"needs_hdd"`
HasDB bool `json:"has_db"`
DrivePath string `json:"drive_path"`
DriveLabel string `json:"drive_label"`
}
// ImportRequest holds user-provided parameters for an import.
type ImportRequest struct {
FABPath string // full path to .fab file
Password string // empty for unencrypted bundles
}
// ScanForBundles scans export directories on all registered drives for .fab files.
// Pass a non-nil logger to enable debug output.
func ScanForBundles(drives []DrivePathInfo) []ScannedBundle {
var bundles []ScannedBundle
for _, drive := range drives {
exportDir := ExportDir(drive.Path)
entries, err := os.ReadDir(exportDir)
if err != nil {
continue
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".fab") {
continue
}
path := filepath.Join(exportDir, entry.Name())
info, err := entry.Info()
if err != nil {
continue
}
bundle := ScannedBundle{
Path: path,
FileName: entry.Name(),
SizeBytes: info.Size(),
SizeHuman: humanizeBytes(info.Size()),
DrivePath: drive.Path,
DriveLabel: drive.Label,
}
// Check encryption
encrypted, _ := IsEncryptedFAB(path)
bundle.Encrypted = encrypted
// Try to read manifest for metadata (unencrypted only)
var manifest *Manifest
if !encrypted {
manifest, _ = ReadManifestFromFAB(path)
}
if manifest != nil {
bundle.AppName = manifest.AppName
bundle.DisplayName = manifest.DisplayName
bundle.ExportedAt = manifest.ExportedAt.Format("2006-01-02 15:04")
bundle.NeedsHDD = manifest.NeedsHDD
bundle.HasDB = manifest.HasDatabase
} else {
// Parse app name from filename: {appname}_{timestamp}.fab
name := strings.TrimSuffix(entry.Name(), ".fab")
if idx := strings.LastIndex(name, "_"); idx > 0 {
bundle.AppName = name[:idx]
} else {
bundle.AppName = name
}
bundle.ExportedAt = info.ModTime().Format("2006-01-02 15:04")
}
bundles = append(bundles, bundle)
}
}
return bundles
}
// ScanForStaleTempFiles finds stale .fab.tmp and .tgz.tmp files in export directories.
func ScanForStaleTempFiles(drives []DrivePathInfo) []string {
var stale []string
for _, drive := range drives {
exportDir := ExportDir(drive.Path)
entries, err := os.ReadDir(exportDir)
if err != nil {
continue
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if strings.HasSuffix(name, ".tmp") || strings.HasSuffix(name, ".tgz.tmp") {
stale = append(stale, filepath.Join(exportDir, name))
}
}
}
return stale
}
// ReadManifestFromFAB reads the manifest from an unencrypted .fab file.
func ReadManifestFromFAB(fabPath string) (*Manifest, error) {
f, err := os.Open(fabPath)
if err != nil {
return nil, err
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("not a valid gzip: %w", err)
}
defer gr.Close()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, fmt.Errorf("manifest.json not found in bundle")
}
if err != nil {
return nil, err
}
if hdr.Name == "manifest.json" {
data, err := io.ReadAll(tr)
if err != nil {
return nil, err
}
return UnmarshalManifest(data)
}
}
}
// ReadManifestFromEncryptedFAB decrypts and reads the manifest from an encrypted .fab file.
func ReadManifestFromEncryptedFAB(fabPath, password string) (*Manifest, error) {
tmpFile, err := os.CreateTemp("", "fab-manifest-*.tgz")
if err != nil {
return nil, err
}
tmpPath := tmpFile.Name()
tmpFile.Close()
defer os.Remove(tmpPath)
if err := DecryptFile(fabPath, tmpPath, password); err != nil {
return nil, util.MsgError("err.appexport.visszafejtes_sikertelen", err)
}
return ReadManifestFromFAB(tmpPath)
}
// StartImport validates and starts an async import. Returns error if blocked.
func (e *Exporter) StartImport(req ImportRequest) error {
e.mu.Lock()
if e.activeJob != nil && e.activeJob.Running {
e.mu.Unlock()
e.debugf("StartImport rejected: another job is already running")
return fmt.Errorf("export or import already in progress")
}
// R-1187 (d): another operation holds the app-stop marker (a restore, a put-back, a stop-first export) — an import
// now would stop and rewrite an app that operation is moving. Refused before anything is read. Pinned by
// TestR1187d_AnImportWaitsForARestore.
if e.stopGuard != nil {
if busy := e.stopGuard.BusyWith(); busy != "" {
e.mu.Unlock()
e.logger.Printf("[WARN] Import: refused — another operation holds the box (%s) (R-1187 d)", busy)
return util.MsgError("err.appexport.import_foglalt", busy)
}
}
// Validate file exists
info, err := os.Stat(req.FABPath)
if err != nil {
e.mu.Unlock()
e.debugf("StartImport rejected: file not found: %s", req.FABPath)
return fmt.Errorf("bundle file not found: %w", err)
}
e.debugf("StartImport: path=%s size=%s encrypted=%v",
req.FABPath, humanizeBytes(info.Size()), req.Password != "")
steps := []Step{
{Label: "Csomag megnyitása", Status: "pending"},
{Label: "Alkalmazás előkészítése", Status: "pending"},
{Label: "Konfiguráció visszaállítása", Status: "pending"},
{Label: "Felhasználói adatok visszaállítása", Status: "pending"},
{Label: "Adatbázis visszaállítása", Status: "pending"},
{Label: "Alkalmazás indítása", Status: "pending"},
}
job := &Job{
StackName: filepath.Base(req.FABPath),
Steps: steps,
Running: true,
JobType: "import",
}
e.activeJob = job
e.mu.Unlock()
go e.executeImport(req, job)
return nil
}
func (e *Exporter) executeImport(req ImportRequest, job *Job) {
importStart := time.Now()
e.logger.Printf("[INFO] [appexport] Import started for %s", filepath.Base(req.FABPath))
e.debugf("=== IMPORT START: path=%s encrypted=%v ===", req.FABPath, req.Password != "")
defer func() {
job.mu.Lock()
job.Running = false
job.Done = true
job.mu.Unlock()
e.debugf("=== IMPORT END: elapsed=%v ===", time.Since(importStart))
}()
step := 0
// --- Step 0: Open bundle ---
job.setStep(step, "running", "")
stepStart := time.Now()
encrypted, _ := IsEncryptedFAB(req.FABPath)
e.debugf("bundle encrypted: %v", encrypted)
tgzPath := req.FABPath
// Decrypt if needed
if encrypted {
if req.Password == "" {
e.failJob(job, step, "Titkosított csomag — jelszó szükséges")
return
}
tmp, err := os.CreateTemp("", "fab-import-*.tgz")
if err != nil {
e.failJob(job, step, fmt.Sprintf("Temp fájl hiba: %v", err))
return
}
tmpPath := tmp.Name()
tmp.Close()
defer os.Remove(tmpPath)
e.debugf("decrypting bundle to %s", tmpPath)
decStart := time.Now()
if err := DecryptFile(req.FABPath, tmpPath, req.Password); err != nil {
e.failJob(job, step, fmt.Sprintf("Visszafejtés sikertelen: %v", err))
return
}
if decInfo, _ := os.Stat(tmpPath); decInfo != nil {
e.debugf("decrypted in %v: %s", time.Since(decStart), humanizeBytes(decInfo.Size()))
}
tgzPath = tmpPath
}
// Extract tar.gz to temp dir
tmpDir, err := os.MkdirTemp("", "felhom-import-*")
if err != nil {
e.failJob(job, step, fmt.Sprintf("Temp könyvtár hiba: %v", err))
return
}
e.debugf("temp dir: %s", tmpDir)
defer os.RemoveAll(tmpDir)
e.debugf("extracting tar.gz from %s", tgzPath)
extractStart := time.Now()
if err := extractTarGz(tgzPath, tmpDir); err != nil {
e.failJob(job, step, fmt.Sprintf("Csomag kicsomagolása sikertelen: %v", err))
return
}
e.debugf("tar.gz extracted in %v", time.Since(extractStart))
// Log extracted contents
if e.debug {
filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
rel, _ := filepath.Rel(tmpDir, path)
if info.IsDir() {
e.debugf(" [dir] %s/", rel)
} else {
e.debugf(" [file] %s (%s)", rel, humanizeBytes(info.Size()))
}
return nil
})
}
// Read manifest
manifestData, err := os.ReadFile(filepath.Join(tmpDir, "manifest.json"))
if err != nil {
e.failJob(job, step, "manifest.json nem található a csomagban")
return
}
manifest, err := UnmarshalManifest(manifestData)
if err != nil {
e.failJob(job, step, fmt.Sprintf("Manifest hiba: %v", err))
return
}
e.debugf("manifest: app=%s display=%s version=%d controller=%s needsHDD=%v",
manifest.AppName, manifest.DisplayName, manifest.Version, manifest.ControllerVer, manifest.NeedsHDD)
e.debugf("manifest: hasDB=%v dbType=%s hasHDD=%v hasVolume=%v totalSize=%s",
manifest.HasDatabase, manifest.DBType, manifest.HasHDDData, manifest.HasVolumeData,
humanizeBytes(manifest.TotalSizeBytes))
e.debugf("manifest: configFiles=%v volumes=%v hddSubdirs=%v",
manifest.ConfigFiles, manifest.VolumeNames, manifest.HDDSubdirs)
// Update job with app info
job.mu.Lock()
job.StackName = manifest.AppName
job.DisplayName = manifest.DisplayName
job.mu.Unlock()
e.logger.Printf("[INFO] Import: opening bundle for %s (%s)", manifest.AppName, manifest.DisplayName)
// v0.125.0 validate-before-destroy (scenario C): every manifest-claimed data tar must be
// present and non-empty BEFORE the app is stopped and BEFORE any volume is removed. Hollow
// bundles (a containerized v<=0.124.0 exporter stranded the tars host-side) land HERE — the
// pre-fix order wiped the volumes first and only then discovered the emptiness.
if err := validateBundleData(tmpDir, manifest); err != nil {
e.failJob(job, step, fmt.Sprintf("A csomag hiányos — az importálás el sem indult, a meglévő alkalmazás érintetlen. (%v) A csomagot valószínűleg egy régebbi (≤0.124.0), konténerben futó vezérlő exportálta — készíts friss exportot.", err))
return
}
// R-1279 (D48 for the import, brief 2026-10-26 fix B): DOWNLOAD FIRST. Every image of the definition that will run
// (the bundle's compose; the live one when the bundle carries none) is on the box BEFORE step 1 removes the running
// app's volumes — or the import is refused here with the app untouched, in the restores' two sentences (a version
// its maker deleted / a busy or unreachable registry). Measured before: the volumes went first and the pull came at
// the database start, so a version that could not be downloaded left the app gone.
if e.prepareImages != nil {
runCompose := filepath.Join(tmpDir, "config", "docker-compose.yml")
if _, serr := os.Stat(runCompose); serr != nil {
if p, ok := e.provider.GetStackComposePath(manifest.AppName); ok {
runCompose = p
}
}
release, perr := e.prepareImages(manifest.AppName, runCompose)
if perr != nil {
e.logger.Printf("[WARN] Import: REFUSED before anything was touched — an image of %s could not be downloaded: %v (R-1279)", manifest.AppName, perr)
e.failJob(job, step, util.ErrText("hu", perr))
return
}
defer release()
}
e.debugf("step 0 (open bundle) done in %v", time.Since(stepStart))
job.setStep(step, "done", "")
step++
// --- Step 1: Prepare app ---
job.setStep(step, "running", "")
stepStart = time.Now()
stacksDir := e.provider.GetStacksBaseDir()
stackDir := filepath.Join(stacksDir, manifest.AppName)
e.debugf("stacksDir=%s stackDir=%s", stacksDir, stackDir)
isDeployed := e.provider.IsStackDeployed(manifest.AppName)
isRunning := false
if isDeployed {
isRunning = e.provider.IsStackRunning(manifest.AppName)
}
e.debugf("existing stack: deployed=%v running=%v", isDeployed, isRunning)
// R-1187 (c) (data-loss hunt 2026-10-20; proven on 9202 2026-10-22): an import over an app that is installed but
// NOT running used to „overwrite config in place" — its volumes were never removed, and `docker cp` only adds and
// overwrites, so the result was the old data MERGED with the bundle's (a stale SQLite -wal beside the bundle's .db
// is a corruption shape), reported as success. Only a running app's volumes are removed first. So this import is
// refused here, before anything is written; the household starts the app (or removes it) and imports again. „Not
// running" is IsStackRunning's view: running or degraded only, so an unhealthy or restarting app is refused too —
// the text says so; it does NOT advise removal, which would lose data newer than the bundle (review 2026-10-23
// night, C3 + N9; the import path for those states is R-1187).
// Pinned by TestR1187c_AStoppedAppIsNotImportedOver.
if isDeployed && !isRunning {
e.logger.Printf("[WARN] Import: refused — %s is installed but not running; importing over it would merge the old data with the bundle's (R-1187 c)", manifest.AppName)
e.failJob(job, step, "Az alkalmazás telepítve van, de nem fut rendben (leállt, vagy hibásan indul), ezért a csomagot nem töltjük vissza: a régi adatai összekeveredhetnének a csomagéval. Az alkalmazás érintetlen. Ha leállt, indítsd el, és próbáld újra; ha hibásan fut, kérd az üzemeltető segítségét — ne távolítsd el, mert az adatai a csomagnál újabbak lehetnek.")
return
}
if isDeployed {
if isRunning {
e.logger.Printf("[INFO] Import: removing existing stack %s (with volumes)", manifest.AppName)
e.debugf("removing existing stack with volumes")
if err := e.provider.RemoveStackVolumes(manifest.AppName); err != nil {
e.logger.Printf("[WARN] Import: failed to remove stack volumes: %v", err)
e.debugf("RemoveStackVolumes failed: %v — trying just StopStack", err)
_ = e.provider.StopStack(manifest.AppName)
} else {
e.debugf("stack volumes removed successfully")
}
} else {
e.debugf("stack is deployed but not running — will overwrite config in place")
}
} else {
e.debugf("creating new stack directory: %s", stackDir)
if err := os.MkdirAll(stackDir, 0755); err != nil {
e.failJob(job, step, fmt.Sprintf("Könyvtár létrehozása sikertelen: %v", err))
return
}
}
e.debugf("step 1 (prepare app) done in %v", time.Since(stepStart))
job.setStep(step, "done", "")
step++
// --- Step 2: Restore config ---
job.setStep(step, "running", "")
stepStart = time.Now()
configDir := filepath.Join(tmpDir, "config")
e.debugf("restoring config from %s → %s", configDir, stackDir)
env, err := e.restoreConfig(configDir, stackDir, manifest)
if err != nil {
e.failJob(job, step, fmt.Sprintf("Konfiguráció visszaállítása sikertelen: %v", err))
return
}
// R-1187 (found live): compose runs below see the controller's path variables, as at every install and start.
env = withImportPathVars(env, e.provider.GetStackNamespaceRoot(manifest.AppName), e.provider.GetImportRoot())
e.debugf("config restored: %d env vars", len(env))
if e.debug {
for k := range env {
e.debugf(" env: %s=%s", k, maskSecret(k, env[k]))
}
}
e.debugf("step 2 (restore config) done in %v", time.Since(stepStart))
job.setStep(step, "done", "")
step++
// --- Step 3: Restore user data ---
job.setStep(step, "running", "")
stepStart = time.Now()
composePath := filepath.Join(stackDir, "docker-compose.yml")
if manifest.HasHDDData {
e.debugf("restoring HDD data: subdirs=%v", manifest.HDDSubdirs)
if err := e.restoreHDDData(tmpDir, manifest, composePath, env); err != nil {
e.failJob(job, step, fmt.Sprintf("HDD adatok visszaállítása sikertelen: %v", err))
return
}
} else {
e.debugf("no HDD data in bundle — skipping")
}
if manifest.HasVolumeData {
e.debugf("restoring Docker volumes: %v", manifest.VolumeNames)
if err := e.restoreVolumeData(tmpDir, manifest); err != nil {
e.failJob(job, step, fmt.Sprintf("Docker volume visszaállítása sikertelen: %v", err))
return
}
} else {
e.debugf("no volume data in bundle — skipping")
}
e.debugf("step 3 (restore user data) done in %v", time.Since(stepStart))
job.setStep(step, "done", "")
step++
// --- Step 4: Restore database ---
job.setStep(step, "running", "")
stepStart = time.Now()
if manifest.HasDatabase {
e.debugf("restoring database: type=%s", manifest.DBType)
if err := e.restoreDatabase(manifest, tmpDir, composePath, stackDir, env); err != nil {
e.failJob(job, step, fmt.Sprintf("Adatbázis visszaállítása sikertelen: %v", err))
return
}
e.debugf("database restored successfully")
} else {
e.debugf("no database in bundle — skipping")
}
e.debugf("step 4 (restore database) done in %v", time.Since(stepStart))
job.setStep(step, "done", "")
step++
// --- Step 5: Start app ---
job.setStep(step, "running", "")
stepStart = time.Now()
// R-1191: the doors first — a family-only app must never start open (the import used to drop its gate).
if err := e.provider.PrepareImportedApp(stackDir, env); err != nil {
e.failJob(job, step, fmt.Sprintf("Alkalmazás előkészítése sikertelen: %v", err))
return
}
e.debugf("starting stack %s", manifest.AppName)
if err := e.provider.StartStack(manifest.AppName); err != nil {
e.failJob(job, step, fmt.Sprintf("Alkalmazás indítása sikertelen: %v", err))
return
}
e.debugf("stack started in %v", time.Since(stepStart))
// Refresh stack list so UI sees the new/updated app
e.debugf("refreshing stack list")
if err := e.provider.RefreshStacks(); err != nil {
e.logger.Printf("[WARN] Import: refresh stacks failed: %v", err)
e.debugf("RefreshStacks error: %v", err)
} else {
e.debugf("stack list refreshed")
}
job.setStep(step, "done", "")
e.logger.Printf("[INFO] Import completed: %s (%s) in %v", manifest.AppName, manifest.DisplayName, time.Since(importStart))
}
// maskSecret masks sensitive env var values for debug logging.
func maskSecret(key, value string) string {
lower := strings.ToLower(key)
if strings.Contains(lower, "password") || strings.Contains(lower, "secret") ||
strings.Contains(lower, "token") || strings.Contains(lower, "key") {
if len(value) <= 3 {
return "***"
}
return value[:2] + "***"
}
return value
}
// restoreConfig extracts config files to the stack directory and re-encrypts app.yaml.
// Returns the plaintext env map for use in subsequent steps.
func (e *Exporter) restoreConfig(configDir, stackDir string, manifest *Manifest) (map[string]string, error) {
entries, err := os.ReadDir(configDir)
if err != nil {
return nil, fmt.Errorf("reading config dir: %w", err)
}
e.debugf("restoreConfig: %d entries in config dir", len(entries))
var env map[string]string
for _, entry := range entries {
if entry.IsDir() {
continue
}
src := filepath.Join(configDir, entry.Name())
dst := filepath.Join(stackDir, entry.Name())
// app.yaml: read plaintext env, then re-encrypt via provider
if entry.Name() == "app.yaml" {
e.debugf("restoreConfig: reading plaintext app.yaml from bundle")
env, err = readPlaintextAppYaml(src)
if err != nil {
return nil, fmt.Errorf("reading app.yaml: %w", err)
}
e.debugf("restoreConfig: app.yaml has %d env vars, re-encrypting via provider", len(env))
if err := e.provider.SaveEncryptedAppConfig(stackDir, env); err != nil {
return nil, fmt.Errorf("saving encrypted app.yaml: %w", err)
}
e.debugf("restoreConfig: app.yaml re-encrypted and saved")
continue
}
// Copy other config files as-is
if info, _ := entry.Info(); info != nil {
e.debugf("restoreConfig: copying %s (%s)", entry.Name(), humanizeBytes(info.Size()))
}
if err := copyFile(src, dst); err != nil {
return nil, fmt.Errorf("copying %s: %w", entry.Name(), err)
}
}
return env, nil
}
// readPlaintextAppYaml parses the plaintext app.yaml written by the export.
func readPlaintextAppYaml(path string) (map[string]string, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
env := make(map[string]string)
inEnv := false
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
continue
}
if trimmed == "env:" {
inEnv = true
continue
}
if inEnv && strings.HasPrefix(line, " ") {
parts := strings.SplitN(trimmed, ":", 2)
if len(parts) == 2 {
key := strings.TrimSpace(parts[0])
val := strings.TrimSpace(parts[1])
// Remove Go-style quotes (written by writeDecryptedAppYaml with %q)
if len(val) >= 2 && val[0] == '"' && val[len(val)-1] == '"' {
if unquoted, err := strconv.Unquote(val); err == nil {
val = unquoted
}
}
env[key] = val
}
} else if inEnv && !strings.HasPrefix(line, " ") {
inEnv = false
}
}
return env, nil
}
// restoreHDDData extracts HDD bind mount data from the bundle.
func (e *Exporter) restoreHDDData(tmpDir string, manifest *Manifest, composePath string, env map[string]string) error {
hddDir := filepath.Join(tmpDir, "data", "hdd")
// Resolve HDD mount paths from compose file
hddPath := env["HDD_PATH"]
e.debugf("restoreHDDData: HDD_PATH=%s composePath=%s", hddPath, composePath)
mountByName := make(map[string]string)
if hddPath != "" {
mounts := resolveHDDMounts(composePath, env)
e.debugf("restoreHDDData: resolved mounts: %v", mounts)
for _, m := range mounts {
mountByName[filepath.Base(m)] = m
}
} else {
e.debugf("restoreHDDData: HDD_PATH is empty — will use fallback paths")
}
for _, subdir := range manifest.HDDSubdirs {
// [CTRL-001] defence-in-depth: refuse any subdir that is not a single
// safe segment before it reaches MkdirAll/extractTar on a user drive.
if err := ValidateSegment("hdd_subdir", subdir); err != nil {
return err
}
tarPath := filepath.Join(hddDir, subdir+".tar")
tarInfo, err := os.Stat(tarPath)
if err != nil {
// v0.125.0: a claimed-but-absent tar is an ERROR (validateBundleData already refused
// it pre-destroy; this is defense in depth, not a reachable soft path).
return fmt.Errorf("HDD tar missing from bundle: %s", subdir+".tar")
}
e.debugf("restoreHDDData: subdir=%s tarSize=%s", subdir, humanizeBytes(tarInfo.Size()))
destPath := mountByName[subdir]
if destPath == "" {
if filepath.Base(hddPath) == subdir {
destPath = hddPath
} else {
destPath = filepath.Join(hddPath, subdir)
}
e.debugf("restoreHDDData: no mount match for %s, using fallback: %s", subdir, destPath)
} else {
e.debugf("restoreHDDData: mount match for %s → %s", subdir, destPath)
}
if err := os.MkdirAll(destPath, 0755); err != nil {
return fmt.Errorf("creating %s: %w", destPath, err)
}
e.logger.Printf("[INFO] Import: extracting HDD data %s → %s", subdir, destPath)
extractStart := time.Now()
refused, err := extractTar(tarPath, destPath)
if err != nil {
return fmt.Errorf("extracting %s: %w", subdir, err)
}
if len(refused) > 0 {
e.logger.Printf("[WARN] Import: %s — %d link(s) pointing outside the app's folder were NOT restored (R-1187 b): %s",
subdir, len(refused), strings.Join(refused, ", "))
}
e.debugf("restoreHDDData: extracted %s in %v", subdir, time.Since(extractStart))
}
return nil
}
// resolveHDDMounts parses compose volumes for HDD bind mounts and resolves env vars.
func resolveHDDMounts(composePath string, env map[string]string) []string {
data, err := os.ReadFile(composePath)
if err != nil {
return nil
}
hddPath := env["HDD_PATH"]
if hddPath == "" {
return nil
}
var mounts []string
seen := make(map[string]bool)
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "- ") {
continue
}
vol := strings.TrimPrefix(trimmed, "- ")
// Parse "host:container" or "host:container:mode"
parts := strings.SplitN(vol, ":", 2)
if len(parts) < 2 {
continue
}
hostPart := parts[0]
if strings.Contains(hostPart, "${HDD_PATH}") {
resolved := strings.ReplaceAll(hostPart, "${HDD_PATH}", hddPath)
resolved = filepath.Clean(resolved)
if !seen[resolved] {
seen[resolved] = true
mounts = append(mounts, resolved)
}
}
}
return mounts
}
// validateBundleData asserts every manifest-claimed data tar exists non-empty in the extracted
// bundle (v0.125.0, scenario C). Pure read — called before ANY destructive import step.
func validateBundleData(tmpDir string, manifest *Manifest) error {
// R-1187 (f): drive data needs an ABSOLUTE, clean HDD_PATH from the bundle's app.yaml — an empty one made the drive
// tars extract into a relative path (the controller's working directory). Refused before anything is touched.
// Pinned by TestR1187f_AnEmptyOrRelativeHDDPathIsRefused.
if len(manifest.HDDSubdirs) > 0 {
env, _ := readPlaintextAppYaml(filepath.Join(tmpDir, "config", "app.yaml"))
hdd := strings.TrimSpace(env["HDD_PATH"])
if hdd == "" || !filepath.IsAbs(hdd) || filepath.Clean(hdd) != hdd || strings.Contains(hdd, "..") {
return util.MsgError("err.appexport.hdd_path_hibas", hdd)
}
}
for _, v := range manifest.VolumeNames {
if err := ValidateSegment("volume_name", v); err != nil {
return err
}
fi, err := os.Stat(filepath.Join(tmpDir, "data", "volumes", v+".tar"))
if err != nil || fi.Size() == 0 {
return util.MsgError("err.appexport.a_z_kotet_adata_hianyzik_a", v)
}
}
for _, s := range manifest.HDDSubdirs {
if err := ValidateSegment("hdd_subdir", s); err != nil {
return err
}
fi, err := os.Stat(filepath.Join(tmpDir, "data", "hdd", s+".tar"))
if err != nil || fi.Size() == 0 {
return util.MsgError("err.appexport.a_z_adatkonyvtar_tartalma_hianyzik_a", s)
}
}
return nil
}
// importVolumeTar streams tarPath into the (existing) volume via `docker cp - <cid>:/vol` —
// zero shared paths, correct on bare metal AND under the containerized controller (the
// docker-run -v population was the v0.124.0 strand's import half).
func (e *Exporter) importVolumeTar(volName, tarPath string) error {
return e.withVolumeHelper(volName, func(cid string) error {
f, err := os.Open(tarPath)
if err != nil {
return fmt.Errorf("opening %s: %w", tarPath, err)
}
defer f.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
if stderr, err := dockerExec(ctx, f, nil, "cp", "-", cid+":/vol"); err != nil {
return fmt.Errorf("streaming into volume %s: %s — %w", volName, stderr, err)
}
return nil
})
}
// restoreVolumeData recreates Docker named volumes from bundle tarballs. v0.125.0: a missing
// tar is an ERROR (defense in depth behind validateBundleData), and population streams via
// docker cp instead of a docker-run -v host mount.
func (e *Exporter) restoreVolumeData(tmpDir string, manifest *Manifest) error {
volDir := filepath.Join(tmpDir, "data", "volumes")
for _, volName := range manifest.VolumeNames {
// [CTRL-001] defence-in-depth: refuse any volume name that is not a
// single safe segment before it reaches a tar path / docker volume op.
if err := ValidateSegment("volume_name", volName); err != nil {
return err
}
tarPath := filepath.Join(volDir, volName+".tar")
tarInfo, err := os.Stat(tarPath)
if err != nil {
return fmt.Errorf("volume tar missing from bundle: %s", volName+".tar")
}
e.debugf("restoreVolumeData: volume=%s tarSize=%s", volName, humanizeBytes(tarInfo.Size()))
// Create the Docker volume
e.debugf("restoreVolumeData: creating docker volume %s", volName)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
stderr, err := dockerExec(ctx, nil, nil, "volume", "create", volName)
cancel()
if err != nil {
return fmt.Errorf("creating volume %s: %s — %w", volName, stderr, err)
}
// Populate volume from tar (docker cp streaming)
e.logger.Printf("[INFO] Import: populating volume %s", volName)
popStart := time.Now()
if err := e.importVolumeTar(volName, tarPath); err != nil {
return fmt.Errorf("populating volume %s: %w", volName, err)
}
e.debugf("restoreVolumeData: volume %s populated in %v", volName, time.Since(popStart))
}
return nil
}
// restoreDatabase imports the database dump from the bundle.
func (e *Exporter) restoreDatabase(manifest *Manifest, tmpDir, composePath, stackDir string, env map[string]string) error {
dbDir := filepath.Join(tmpDir, "database")
// Find the gzipped dump file
dumpGzPath, err := findDumpFile(dbDir)
if err != nil {
e.debugf("restoreDatabase: no dump file found in %s: %v", dbDir, err)
return err
}
if dumpInfo, _ := os.Stat(dumpGzPath); dumpInfo != nil {
e.debugf("restoreDatabase: dump file=%s size=%s", dumpGzPath, humanizeBytes(dumpInfo.Size()))
}
// Find DB service in compose file
e.debugf("restoreDatabase: parsing compose for DB service: %s", composePath)
dbSvc := findDBServiceInCompose(composePath)
if dbSvc == nil {
e.debugf("restoreDatabase: no DB service found in compose file!")
return fmt.Errorf("no database service found in docker-compose.yml")
}
e.debugf("restoreDatabase: found DB service=%s type=%s container=%s",
dbSvc.ServiceName, dbSvc.DBType, dbSvc.ContainerName)
// Start only the DB service
e.logger.Printf("[INFO] Import: starting DB service %s (%s)", dbSvc.ServiceName, dbSvc.DBType)
e.debugf("restoreDatabase: docker compose up -d %s", dbSvc.ServiceName)
out, err := composeExecEnv(stackDir, env, "up", "-d", dbSvc.ServiceName)
if err != nil {
e.debugf("restoreDatabase: compose up failed: %s", strings.TrimSpace(string(out)))
return fmt.Errorf("starting DB service: %w", err)
}
e.debugf("restoreDatabase: compose up output: %s", strings.TrimSpace(string(out)))
// Ensure we stop the DB service after restore (full stack start is in the next step)
defer func() {
e.debugf("restoreDatabase: stopping DB service %s", dbSvc.ServiceName)
out, _ := composeExecEnv(stackDir, env, "stop", dbSvc.ServiceName)
e.debugf("restoreDatabase: compose stop output: %s", strings.TrimSpace(string(out)))
}()
// Get the container ID
e.debugf("restoreDatabase: getting container ID for service %s", dbSvc.ServiceName)
containerID, err := getComposeContainerID(stackDir, env, dbSvc.ServiceName)
if err != nil {
e.debugf("restoreDatabase: failed to get container ID: %v", err)
return fmt.Errorf("getting container ID: %w", err)
}
e.debugf("restoreDatabase: container ID=%s", containerID)
// Wait for DB readiness
e.logger.Printf("[INFO] Import: waiting for DB readiness (container %s)", containerID[:12])
e.debugf("restoreDatabase: waiting for DB readiness...")
waitStart := time.Now()
if err := waitForDB(containerID, dbSvc.DBType, env); err != nil {
e.debugf("restoreDatabase: DB not ready after %v: %v", time.Since(waitStart), err)
return fmt.Errorf("waiting for DB: %w", err)
}
e.debugf("restoreDatabase: DB ready in %v", time.Since(waitStart))
// Import the dump
e.logger.Printf("[INFO] Import: importing database dump")
e.debugf("restoreDatabase: importing dump into container %s", containerID[:12])
importStart := time.Now()
if err := importDBDump(containerID, dumpGzPath, dbSvc.DBType, env); err != nil {
e.debugf("restoreDatabase: import failed after %v: %v", time.Since(importStart), err)
return fmt.Errorf("importing dump: %w", err)
}
e.debugf("restoreDatabase: dump imported in %v", time.Since(importStart))
return nil
}
// findDumpFile finds the gzipped SQL dump in the database directory.
func findDumpFile(dbDir string) (string, error) {
entries, err := os.ReadDir(dbDir)
if err != nil {
return "", fmt.Errorf("reading database dir: %w", err)
}
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".sql.gz") || strings.HasSuffix(entry.Name(), ".sql") {
return filepath.Join(dbDir, entry.Name()), nil
}
}
return "", fmt.Errorf("no database dump file found")
}
// dbServiceInfo describes a database service found in a compose file.
type dbServiceInfo struct {
ServiceName string
DBType string // "postgres" or "mariadb"
ContainerName string
}
// findDBServiceInCompose parses a compose file to find the database service.
func findDBServiceInCompose(composePath string) *dbServiceInfo {
data, err := os.ReadFile(composePath)
if err != nil {
return nil
}
inServices := false
currentService := ""
currentImage := ""
currentContainer := ""
checkAndReturn := func() *dbServiceInfo {
if currentService != "" && currentImage != "" {
return classifyDBImage(currentService, currentImage, currentContainer)
}
return nil
}
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
// Top-level section detection
if len(line) > 0 && line[0] != ' ' && line[0] != '#' {
if strings.HasPrefix(trimmed, "services:") {
inServices = true
} else {
// Leaving services section — check last service
if info := checkAndReturn(); info != nil {
return info
}
inServices = false
}
continue
}
if !inServices {
continue
}
// Service name: 2-space indent, ends with colon, no further nesting
if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") &&
strings.HasSuffix(trimmed, ":") && !strings.Contains(trimmed, " ") {
// Check previous service
if info := checkAndReturn(); info != nil {
return info
}
currentService = strings.TrimSuffix(trimmed, ":")
currentImage = ""
currentContainer = ""
continue
}
// Properties at 4+ space indent
if strings.HasPrefix(trimmed, "image:") {
currentImage = strings.TrimSpace(strings.TrimPrefix(trimmed, "image:"))
}
if strings.HasPrefix(trimmed, "container_name:") {
currentContainer = strings.TrimSpace(strings.TrimPrefix(trimmed, "container_name:"))
}
}
// Check last service
return checkAndReturn()
}
func classifyDBImage(service, image, container string) *dbServiceInfo {
// R-1187 (g): the backup side's predicate (appbackup.DBTypeForImage, R-484) — postgis / pgvector / timescaledb are
// PostgreSQL too. The import used its own narrower list and skipped adventurelog's and dawarich's database.
switch t, ok := appbackup.DBTypeForImage(image); {
case ok && t == appbackup.DBTypePostgres:
return &dbServiceInfo{ServiceName: service, DBType: "postgres", ContainerName: container}
case ok && t == appbackup.DBTypeMariaDB:
return &dbServiceInfo{ServiceName: service, DBType: "mariadb", ContainerName: container}
}
return nil
}
// composeExecEnv runs docker compose in the given stack directory with env vars.
func composeExecEnv(stackDir string, env map[string]string, args ...string) ([]byte, error) {
defer dockerexec.BeginImageWork(args)() // R-863: no image clean-up while a restore may pull
cmdArgs := append([]string{"compose"}, args...)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
cmd := dockerexec.CommandContext(ctx, "docker", cmdArgs...)
cmd.Dir = stackDir
cmd.Env = os.Environ()
for k, v := range env {
cmd.Env = append(cmd.Env, k+"="+v)
}
return cmd.CombinedOutput()
}
// getComposeContainerID returns the container ID for a compose service.
func getComposeContainerID(stackDir string, env map[string]string, service string) (string, error) {
// Wait a moment for the container to be created
time.Sleep(2 * time.Second)
out, err := composeExecEnv(stackDir, env, "ps", "-q", service)
if err != nil {
return "", fmt.Errorf("compose ps: %w", err)
}
id := containerIDFrom(out)
if id == "" {
return "", fmt.Errorf("no container found for service %s", service)
}
return id, nil
}
// containerIDFrom picks the container id out of `compose ps -q` output: the first line that IS an id (12–64 hex). The
// output is combined, so a compose WARNING can come first — it was read as the id once (R-1187, found live on 9202).
func containerIDFrom(out []byte) string {
for _, line := range strings.Split(string(out), "\n") {
line = strings.TrimSpace(line)
if n := len(line); n >= 12 && n <= 64 && strings.Trim(line, "0123456789abcdef") == "" {
return line
}
}
return ""
}
// withImportPathVars is the import's env as `compose` must see it: the bundle's values plus the two path variables the
// controller itself adds at every install and start (stacks.policyEnv) — USERDATA_PATH always the controller's own
// derivation from the namespace root (never a bundle's value), IMPORT_PATH the drop-zone root. Without them compose
// printed a warning the DB step read as the container id (R-1187, live on 9202). Returns a new map.
func withImportPathVars(env map[string]string, nsRoot, importRoot string) map[string]string {
out := make(map[string]string, len(env)+2)
for k, v := range env {
out[k] = v
}
delete(out, "USERDATA_PATH")
delete(out, "IMPORT_PATH")
if nsRoot != "" {
out["USERDATA_PATH"] = appbackup.UserdataDir(nsRoot)
}
if importRoot != "" {
out["IMPORT_PATH"] = importRoot
}
return out
}
// waitForDB polls until the database is ready to accept connections.
func waitForDB(containerID, dbType string, env map[string]string) error {
timeout := 60 * time.Second
start := time.Now()
for {
if time.Since(start) > timeout {
return fmt.Errorf("timeout waiting for %s to become ready", dbType)
}
var cmd *exec.Cmd
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
switch dbType {
case "postgres":
user := env["POSTGRES_USER"]
if user == "" {
user = "postgres"
}
cmd = dockerexec.CommandContext(ctx, "docker", "exec", containerID,
"pg_isready", "-U", user)
case "mariadb":
password := env["MYSQL_ROOT_PASSWORD"]
if password == "" {
password = env["MARIADB_ROOT_PASSWORD"]
}
cmd = dockerexec.CommandContext(ctx, "docker", "exec", containerID,
"mysqladmin", "ping", "-u", "root", "-p"+password)
default:
cancel()
return fmt.Errorf("unknown DB type: %s", dbType)
}
err := cmd.Run()
cancel()
if err == nil {
return nil
}
time.Sleep(2 * time.Second)
}
}
// psqlImportArgs is the `docker` argv that loads a Postgres dump.
func psqlImportArgs(containerID, user, dbName string) []string {
// R-1187 (g): stop on the first error — the dump is --clean --if-exists, so a failing statement is a real failure,
// and without ON_ERROR_STOP psql went on and the import reported success over a half-loaded database.
return []string{"exec", "-i", containerID, "psql", "--set=ON_ERROR_STOP=1", "-U", user, "-d", dbName}
}
// importDBDump imports a (possibly gzipped) SQL dump into the running database container.
func importDBDump(containerID, dumpPath, dbType string, env map[string]string) error {
// Open the dump file
f, err := os.Open(dumpPath)
if err != nil {
return err
}
defer f.Close()
// If gzipped, wrap in gzip reader
var reader io.Reader = f
if strings.HasSuffix(dumpPath, ".gz") {
gr, err := gzip.NewReader(f)
if err != nil {
return fmt.Errorf("opening gzip: %w", err)
}
defer gr.Close()
reader = gr
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
var cmd *exec.Cmd
switch dbType {
case "postgres":
user := env["POSTGRES_USER"]
if user == "" {
user = "postgres"
}
dbName := env["POSTGRES_DB"]
if dbName == "" {
dbName = user
}
cmd = dockerexec.CommandContext(ctx, "docker", psqlImportArgs(containerID, user, dbName)...)
case "mariadb":
password := env["MYSQL_ROOT_PASSWORD"]
if password == "" {
password = env["MARIADB_ROOT_PASSWORD"]
}
dbName := env["MYSQL_DATABASE"]
if dbName == "" {
dbName = env["MARIADB_DATABASE"]
}
cmd = dockerexec.CommandContext(ctx, "docker", "exec", "-i", containerID,
"mysql", "-u", "root", "-p"+password, dbName)
default:
return fmt.Errorf("unknown DB type: %s", dbType)
}
cmd.Stdin = reader
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%s import failed: %s — %w", dbType, strings.TrimSpace(string(out)), err)
}
return nil
}
// extractTarGz extracts a tar.gz archive to a directory.
func extractTarGz(tgzPath, destDir string) error {
f, err := os.Open(tgzPath)
if err != nil {
return err
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gr.Close()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
target := filepath.Join(destDir, hdr.Name)
// Security: prevent path traversal
if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) {
continue
}
switch hdr.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
outFile, err := os.Create(target)
if err != nil {
return err
}
if _, err := io.Copy(outFile, tr); err != nil {
outFile.Close()
return err
}
outFile.Close()
os.Chmod(target, os.FileMode(hdr.Mode))
}
}
}
// safeDownwardLink is the rule for a restored link: a relative target with no „.." component, and no symbolic link on
// the way — not among the folders of the link's own path, not among the existing components of its target.
func safeDownwardLink(rt *os.Root, name, target string) bool {
if target == "" || filepath.IsAbs(target) {
return false
}
for _, part := range strings.Split(filepath.ToSlash(target), "/") {
if part == ".." {
return false
}
}
if parentThroughLink(rt, name) {
return false
}
acc := filepath.Dir(name)
for _, part := range strings.Split(filepath.Clean(target), string(os.PathSeparator)) {
acc = filepath.Join(acc, part)
fi, err := rt.Lstat(acc)
if err != nil {
break // not there (yet): nothing further down can be a link that exists now
}
if fi.Mode()&os.ModeSymlink != 0 {
return false
}
}
return true
}
// parentThroughLink is true when any folder on name's path (inside the root) is a symbolic link.
func parentThroughLink(rt *os.Root, name string) bool {
dir := filepath.Dir(name)
if dir == "." {
return false
}
acc := ""
for _, part := range strings.Split(dir, string(os.PathSeparator)) {
acc = filepath.Join(acc, part)
fi, err := rt.Lstat(acc)
if err != nil {
return false // not there yet: MkdirAll creates real folders
}
if fi.Mode()&os.ModeSymlink != 0 {
return true
}
}
return false
}
// lchownFn is the owner seam (a test records it; production is os.Lchown).
var lchownFn = func(r *os.Root, name string, uid, gid int) error { return r.Lchown(name, uid, gid) }
// extractTar extracts a plain tar archive (one drive folder) into destDir and returns the entries it REFUSED.
//
// R-1187 (b): owners and modes are kept (Lchown — the controller runs as root on a box), and a symlink is restored when
// it stays inside destDir; a link pointing outside (absolute, or climbing out with ..) is refused and listed — never
// followed, never created. Only folders, regular files and links exist in a drive tar (the export refuses the rest).
// Pinned by TestR1187b_TheImportKeepsOwnersModesAndInsideLinks.
func extractTar(tarPath, destDir string) ([]string, error) {
f, err := os.Open(tarPath)
if err != nil {
return nil, err
}
defer f.Close()
root := filepath.Clean(destDir)
// Every write goes through os.Root: no entry can create or write a file outside destDir, not even through a link
// already in the folder (left by the old app, or made by an earlier entry) that points out — the security review's
// case, pinned by TestR1187b_NoWriteThroughALinkThatLeavesTheFolder.
rt, err := os.OpenRoot(root)
if err != nil {
return nil, err
}
defer rt.Close()
var refused []string
tr := tar.NewReader(f)
for {
hdr, err := tr.Next()
if err == io.EOF {
return refused, nil
}
if err != nil {
return refused, err
}
name := filepath.Clean(hdr.Name)
if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(os.PathSeparator)) {
continue // path traversal
}
switch hdr.Typeflag {
case tar.TypeDir:
if err := rt.MkdirAll(name, os.FileMode(hdr.Mode).Perm()); err != nil {
return refused, err
}
rt.Chmod(name, os.FileMode(hdr.Mode).Perm())
case tar.TypeReg:
if dir := filepath.Dir(name); dir != "." {
if err := rt.MkdirAll(dir, 0755); err != nil {
return refused, err
}
}
out, err := rt.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(hdr.Mode).Perm())
if err != nil {
refused = append(refused, hdr.Name) // e.g. its path goes through a link that leaves the folder
continue
}
if _, err := io.Copy(out, tr); err != nil {
out.Close()
return refused, err
}
out.Close()
rt.Chmod(name, os.FileMode(hdr.Mode).Perm())
case tar.TypeSymlink:
// A restored link may only point DOWNWARD inside the folder, through real folders: no absolute target, no
// „..", and no link on the way — neither on the link's own path nor in its target. Such a link can never
// resolve outside the app's folder, whatever else the folder holds. Everything else is refused and listed.
// (Three rounds of the automated security review: a link written through, a linked parent, a target through
// a link. Pinned by TestR1187b_*.)
if !safeDownwardLink(rt, name, hdr.Linkname) {
refused = append(refused, hdr.Name)
continue
}
dst := hdr.Linkname
if dir := filepath.Dir(name); dir != "." {
if err := rt.MkdirAll(dir, 0755); err != nil {
return refused, err
}
}
rt.Remove(name)
if err := rt.Symlink(dst, name); err != nil {
refused = append(refused, hdr.Name)
continue
}
default:
refused = append(refused, hdr.Name)
continue
}
if err := lchownFn(rt, name, hdr.Uid, hdr.Gid); err != nil && !os.IsPermission(err) {
return refused, fmt.Errorf("setting the owner of %s: %w", hdr.Name, err)
}
}
}