7c05b59708
gates / gates (push) Successful in 24s
179 Hungarian sentences were built deep inside a package with fmt.Errorf and printed by whoever caught them: too late to translate where they are shown, too early where they are made. Every one now carries its key across that gap. ZERO Hungarian error literals remain. util.MsgError does three things at once, each earned: - Error() is the Hungarian, byte for byte, so every un-converted printer is unchanged; - errors.Is answers for the kind AND for a wrapped cause (KindErrorf dropped the cause); - an error ARGUMENT renders recursively, so "formázás sikertelen: %w" translates whole. A foreign error — restic, docker, ssh, the stdlib — prints verbatim. It is not ours. 76 display sites go through errText, and TestNoErrErrorInPageOutput convicts any that do not. memoryVerdict returns an error rather than a sentence, so the deploy's 409 and the household's language come from one value; UpdateRefusal gained a Cause to carry it. Plurals, one rule, stated once: a key with .one/.other takes its COUNT first. Not a per-call-site flag — the producer somebody forgot would read "3 app is not running". The guard caught a real key collision (alert.deadapp.one) the day the rule landed. TWO DEFECTS FOUND IN MY OWN TOOLING, recorded rather than quietly fixed. The bulk converter silently dropped multi-line concatenations, damaging 7 producers — and the parity gate could not see it, because every surviving fragment WAS a real base literal while the CALL had lost text; two behaviour tests caught it. And the counting script was case-sensitive, so it said "0 left" while five remained. MinAgent: 0.131.0 (unchanged). No hub release needed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
965 lines
32 KiB
Go
965 lines
32 KiB
Go
package appexport
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
|
)
|
|
|
|
// Step tracks one step of an export/import operation.
|
|
type Step struct {
|
|
Label string `json:"label"`
|
|
Status string `json:"status"` // "pending", "running", "done", "failed"
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// Job tracks an in-progress export or import operation.
|
|
type Job struct {
|
|
mu sync.RWMutex
|
|
|
|
StackName string `json:"stack_name"`
|
|
DisplayName string `json:"display_name"`
|
|
Steps []Step `json:"steps"`
|
|
Running bool `json:"running"`
|
|
Done bool `json:"done"`
|
|
Error string `json:"error,omitempty"`
|
|
OutputPath string `json:"output_path,omitempty"`
|
|
OutputSize string `json:"output_size,omitempty"`
|
|
JobType string `json:"job_type"` // "export" or "import"
|
|
}
|
|
|
|
// Snapshot returns a thread-safe copy for JSON serialization.
|
|
func (j *Job) Snapshot() map[string]interface{} {
|
|
j.mu.RLock()
|
|
defer j.mu.RUnlock()
|
|
steps := make([]Step, len(j.Steps))
|
|
copy(steps, j.Steps)
|
|
return map[string]interface{}{
|
|
"ok": true,
|
|
"running": j.Running,
|
|
"done": j.Done,
|
|
"error": j.Error,
|
|
"steps": steps,
|
|
"output_path": j.OutputPath,
|
|
"output_size": j.OutputSize,
|
|
"stack_name": j.StackName,
|
|
"display_name": j.DisplayName,
|
|
"job_type": j.JobType,
|
|
}
|
|
}
|
|
|
|
func (j *Job) setStep(idx int, status, errMsg string) {
|
|
j.mu.Lock()
|
|
defer j.mu.Unlock()
|
|
if idx < len(j.Steps) {
|
|
j.Steps[idx].Status = status
|
|
j.Steps[idx].Error = errMsg
|
|
}
|
|
}
|
|
|
|
// ExportRequest holds user-provided parameters for an export.
|
|
type ExportRequest struct {
|
|
StackName string
|
|
DestDrive string // drive mount path (e.g., "/mnt/hdd_1")
|
|
Password string // empty = no encryption
|
|
StopApp bool // stop app before export
|
|
|
|
// `.fab` class-scoped selection (Task 4; classified apps only — legacy apps ignore these). Both
|
|
// empty = ruling #1 defaults (mandatory in, optional in, excluded out). Values are "root/rel" keys
|
|
// (matching a CapturePath: "hdd/appdata/x" | "userdata/media/y"). The server enforces the floor:
|
|
// a DeselectOptional entry naming a MANDATORY path is ignored with a WARN (the client cannot weaken
|
|
// the mandatory floor). OptInExcluded pulls an excluded bind into the bundle.
|
|
DeselectOptional []string
|
|
OptInExcluded []string
|
|
}
|
|
|
|
// Exporter manages app export/import operations.
|
|
type Exporter struct {
|
|
provider ExportStackProvider
|
|
logger *log.Logger
|
|
version string
|
|
debug bool
|
|
|
|
// dirLister (Task 4) lists child DIR names of a path — the seam the `.fab` userdata-exclude
|
|
// computation walks. Nil → the real os.ReadDir-based lister.
|
|
dirLister func(dir string) []string
|
|
|
|
// stopGuard (R-166) marks the stop→export→start window so a controller killed inside it leaves a
|
|
// durable record that the app is owed a restart. Declared consumer-side as a two-method interface
|
|
// so this package does not import internal/backup; main.go passes the backup manager's guard, so
|
|
// BOTH packages write ONE marker file — an exporter with its own file would be a second writer
|
|
// racing the same recovery. Nil = not wired (tests): the export runs exactly as it did before.
|
|
stopGuard appStopGuard
|
|
|
|
mu sync.Mutex
|
|
activeJob *Job
|
|
}
|
|
|
|
// appStopGuard is the app-stop crash-marker seam. The REASON is deliberately not a parameter: it is
|
|
// always "app export" from here, and the adapter in main.go supplies it. Passing it as a string
|
|
// would duplicate backup.ReasonAppExport's value in a second package with nothing keeping the two in
|
|
// step — a drift this codebase has paid for before (the offbox key that was guessed, R-7b).
|
|
type appStopGuard interface {
|
|
Begin(opID string, stacks []string) error
|
|
End()
|
|
// ReleaseFailed (R-330) drops the app-down alarm suppression for a stack this export stopped and
|
|
// then could not restart. Part of the seam rather than the exporter's own bookkeeping for the
|
|
// same reason Begin is: the guard owns the "we are holding this app" fact, and a second owner
|
|
// would drift from it.
|
|
ReleaseFailed(stacks ...string)
|
|
}
|
|
|
|
// NewExporter creates a new export/import engine.
|
|
func NewExporter(provider ExportStackProvider, logger *log.Logger, version string) *Exporter {
|
|
return &Exporter{
|
|
provider: provider,
|
|
logger: logger,
|
|
version: version,
|
|
}
|
|
}
|
|
|
|
// SetStopGuard wires the app-stop crash marker. INIT-ONLY — call once at startup, before any export.
|
|
func (e *Exporter) SetStopGuard(g appStopGuard) { e.stopGuard = g }
|
|
|
|
// stopGuardBegin records the app-stop marker before an export stops an app. An unwired guard is a
|
|
// no-op (pre-v0.189.0 behaviour), never an error — a test exporter must not be forced to have one.
|
|
func (e *Exporter) stopGuardBegin(stackName string) error {
|
|
if e.stopGuard == nil {
|
|
return nil
|
|
}
|
|
return e.stopGuard.Begin("app-export:"+stackName, []string{stackName})
|
|
}
|
|
|
|
// SetDebug enables or disables verbose debug logging.
|
|
func (e *Exporter) SetDebug(debug bool) {
|
|
e.debug = debug
|
|
}
|
|
|
|
// debugf logs a message only when debug mode is enabled.
|
|
func (e *Exporter) debugf(format string, args ...interface{}) {
|
|
if e.debug {
|
|
e.logger.Printf("[DEBUG] [appexport] "+format, args...)
|
|
}
|
|
}
|
|
|
|
// IsRunning returns true if an export/import is in progress.
|
|
func (e *Exporter) IsRunning() bool {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
return e.activeJob != nil && e.activeJob.Running
|
|
}
|
|
|
|
// GetActiveJob returns the current job (for status polling).
|
|
func (e *Exporter) GetActiveJob() *Job {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
return e.activeJob
|
|
}
|
|
|
|
// StartExport validates and starts an async export. Returns error if blocked.
|
|
func (e *Exporter) StartExport(req ExportRequest) error {
|
|
e.mu.Lock()
|
|
if e.activeJob != nil && e.activeJob.Running {
|
|
e.mu.Unlock()
|
|
e.debugf("StartExport rejected: another job is already running")
|
|
return fmt.Errorf("export or import already in progress")
|
|
}
|
|
|
|
if !e.provider.IsStackDeployed(req.StackName) {
|
|
e.mu.Unlock()
|
|
e.debugf("StartExport rejected: stack %q is not deployed", req.StackName)
|
|
return fmt.Errorf("stack %q is not deployed", req.StackName)
|
|
}
|
|
|
|
e.debugf("StartExport: stack=%q dest=%q password=%v stopApp=%v",
|
|
req.StackName, req.DestDrive, req.Password != "", req.StopApp)
|
|
|
|
steps := []Step{
|
|
{Label: "Előkészítés", Status: "pending"},
|
|
{Label: "Konfiguráció mentése", Status: "pending"},
|
|
{Label: "Adatbázis mentése", Status: "pending"},
|
|
{Label: "Felhasználói adatok", Status: "pending"},
|
|
{Label: "Csomag készítése", Status: "pending"},
|
|
}
|
|
if req.Password != "" {
|
|
steps = append(steps, Step{Label: "Titkosítás", Status: "pending"})
|
|
}
|
|
|
|
job := &Job{
|
|
StackName: req.StackName,
|
|
DisplayName: e.provider.GetStackDisplayName(req.StackName),
|
|
Steps: steps,
|
|
Running: true,
|
|
JobType: "export",
|
|
}
|
|
e.activeJob = job
|
|
e.mu.Unlock()
|
|
|
|
go e.executeExport(req, job)
|
|
return nil
|
|
}
|
|
|
|
func (e *Exporter) executeExport(req ExportRequest, job *Job) {
|
|
exportStart := time.Now()
|
|
e.debugf("=== EXPORT START: stack=%q dest=%q encrypted=%v stopApp=%v ===",
|
|
req.StackName, req.DestDrive, req.Password != "", req.StopApp)
|
|
|
|
defer func() {
|
|
job.mu.Lock()
|
|
job.Running = false
|
|
job.Done = true
|
|
job.mu.Unlock()
|
|
e.debugf("=== EXPORT END: stack=%q elapsed=%v ===", req.StackName, time.Since(exportStart))
|
|
}()
|
|
|
|
step := 0
|
|
|
|
// --- Step 0: Preparation ---
|
|
job.setStep(step, "running", "")
|
|
stepStart := time.Now()
|
|
|
|
destDir := ExportDir(req.DestDrive)
|
|
e.debugf("export dest dir: %s", destDir)
|
|
if err := os.MkdirAll(destDir, 0755); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Nem sikerült létrehozni az export könyvtárat: %v", err))
|
|
return
|
|
}
|
|
|
|
// Check free space
|
|
est, err := e.EstimateExport(req.StackName, req.DestDrive)
|
|
if err != nil {
|
|
e.debugf("estimate error (non-fatal): %v", err)
|
|
} else {
|
|
e.debugf("estimate: config=%s data=%s total=%s destFree=%s fits=%v unknown=%v",
|
|
est.ConfigSizeHuman, est.DataSizeHuman, est.TotalSizeHuman, est.DestFreeHuman, est.FitsOnDest, est.SizeUnknown)
|
|
// Hard-abort only on a KNOWN doesn't-fit. F-A: est.SizeUnknown forces FitsOnDest=false for
|
|
// the UI honesty signal, but an unmeasured size must NOT block the export here — the tar
|
|
// streaming and the destination filesystem surface a real ENOSPC if it genuinely won't fit.
|
|
if est.SizeUnknown {
|
|
e.logger.Printf("[WARN] appexport: export space pre-check skipped for %s — volume size unknown", req.StackName)
|
|
} else if !est.FitsOnDest {
|
|
e.failJob(job, step, fmt.Sprintf("Nincs elég hely: szükséges ~%s, szabad %s",
|
|
est.TotalSizeHuman, est.DestFreeHuman))
|
|
return
|
|
}
|
|
}
|
|
|
|
// Optionally stop the app
|
|
wasRunning := false
|
|
if req.StopApp && e.provider.IsStackRunning(req.StackName) {
|
|
// R-166: mark BEFORE the stop. The defer below covers the graceful exits; it does NOT cover a
|
|
// SIGKILL or a power cut, which run no deferred function (Campaign 8 fault 10, on live
|
|
// hardware) — only this marker does, and a big export is a long window to be killed in.
|
|
if err := e.stopGuardBegin(req.StackName); err != nil {
|
|
e.failJob(job, step, "Az alkalmazás leállítása előtti jelölő nem menthető — az exportálás nem indult el.")
|
|
e.logger.Printf("[ERROR] Export: could not record the app-stop marker for %s (refusing to stop it unprotected): %v", req.StackName, err)
|
|
return
|
|
}
|
|
wasRunning = true
|
|
e.logger.Printf("[INFO] Export: stopping %s", req.StackName)
|
|
e.debugf("stopping stack %s before export", req.StackName)
|
|
if err := e.provider.StopStack(req.StackName); err != nil {
|
|
e.logger.Printf("[WARN] Export: could not stop %s: %v", req.StackName, err)
|
|
} else {
|
|
e.debugf("stack %s stopped successfully", req.StackName)
|
|
}
|
|
} else {
|
|
e.debugf("skip stop: stopApp=%v isRunning=%v", req.StopApp, e.provider.IsStackRunning(req.StackName))
|
|
}
|
|
// Always restart after export if we stopped it
|
|
if wasRunning {
|
|
defer func() {
|
|
e.logger.Printf("[INFO] Export: restarting %s", req.StackName)
|
|
e.debugf("restarting stack %s after export", req.StackName)
|
|
if err := e.provider.StartStack(req.StackName); err != nil {
|
|
e.logger.Printf("[WARN] Export: could not restart %s: %v", req.StackName, err)
|
|
// R-330: the app is genuinely down now, so the app-down alarm must see it on the very
|
|
// next scan. The marker is still KEPT (above) so the next startup retries — the two
|
|
// are independent: the marker is durable recovery, this is live alarm suppression.
|
|
if e.stopGuard != nil {
|
|
e.stopGuard.ReleaseFailed(req.StackName)
|
|
}
|
|
} else {
|
|
e.debugf("stack %s restarted successfully", req.StackName)
|
|
// Cleared only on a restart that succeeded — a failed one keeps the marker so the
|
|
// next startup retries.
|
|
if e.stopGuard != nil {
|
|
e.stopGuard.End()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
e.debugf("step 0 (preparation) done in %v", time.Since(stepStart))
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 1: Config files ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
tmpDir, err := os.MkdirTemp("", "felhom-export-*")
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Temp könyvtár hiba: %v", err))
|
|
return
|
|
}
|
|
e.debugf("temp dir: %s", tmpDir)
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
configDir := filepath.Join(tmpDir, "config")
|
|
if err := os.MkdirAll(configDir, 0755); err != nil {
|
|
e.failJob(job, step, err.Error())
|
|
return
|
|
}
|
|
|
|
stackDir, ok := e.provider.GetStackDir(req.StackName)
|
|
if !ok {
|
|
e.failJob(job, step, "Stack könyvtár nem található")
|
|
return
|
|
}
|
|
e.debugf("stack dir: %s", stackDir)
|
|
|
|
configFiles, err := copyStackConfig(stackDir, configDir, req.StackName, e.provider)
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Konfiguráció mentése sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("config files copied: %v (%d files)", configFiles, len(configFiles))
|
|
e.debugf("step 1 (config) done in %v", time.Since(stepStart))
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 2: Database dump ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
dbDir := filepath.Join(tmpDir, "database")
|
|
os.MkdirAll(dbDir, 0755)
|
|
|
|
manifest := &Manifest{
|
|
Version: ManifestVersion,
|
|
AppName: req.StackName,
|
|
DisplayName: e.provider.GetStackDisplayName(req.StackName),
|
|
ExportedAt: time.Now().UTC(),
|
|
ControllerVer: e.version,
|
|
NeedsHDD: e.provider.GetStackNeedsHDD(req.StackName),
|
|
Encrypted: req.Password != "",
|
|
ConfigFiles: configFiles,
|
|
}
|
|
e.debugf("manifest: app=%s display=%s needsHDD=%v encrypted=%v",
|
|
manifest.AppName, manifest.DisplayName, manifest.NeedsHDD, manifest.Encrypted)
|
|
|
|
dbDumped := e.dumpDatabase(req.StackName, dbDir, manifest)
|
|
if !dbDumped {
|
|
e.debugf("no database found for %s — skipping DB step", req.StackName)
|
|
os.Remove(dbDir)
|
|
} else {
|
|
e.debugf("database dumped: type=%s", manifest.DBType)
|
|
// Log the dump file size
|
|
entries, _ := os.ReadDir(dbDir)
|
|
for _, entry := range entries {
|
|
if info, err := entry.Info(); err == nil {
|
|
e.debugf(" db dump file: %s (%s)", entry.Name(), humanizeBytes(info.Size()))
|
|
}
|
|
}
|
|
}
|
|
e.debugf("step 2 (database) done in %v", time.Since(stepStart))
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 3: User data ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
dataDir := filepath.Join(tmpDir, "data")
|
|
os.MkdirAll(dataDir, 0755)
|
|
|
|
// C6B-F1 cause 1 (v0.130.0): user-data capture is ADDITIVE, not either/or. A needs_hdd app
|
|
// can hold state in BOTH its HDD binds and its named volumes (sonarr: ${USERDATA_PATH} media
|
|
// binds + the sonarr_config volume with the entire app DB) — the old else-branch silently
|
|
// dropped every named volume of every needs_hdd app.
|
|
if e.provider.GetStackNeedsHDD(req.StackName) {
|
|
e.debugf("exporting HDD data for %s", req.StackName)
|
|
if err := e.exportHDDData(req, dataDir, manifest); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Felhasználói adatok mentése sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("HDD data exported: subdirs=%v hasData=%v", manifest.HDDSubdirs, manifest.HasHDDData)
|
|
}
|
|
e.debugf("exporting Docker volumes for %s", req.StackName)
|
|
if err := e.exportVolumeData(req.StackName, dataDir, manifest); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Kötet mentése sikertelen: %v", err))
|
|
return
|
|
}
|
|
e.debugf("volume data exported: volumes=%v hasData=%v", manifest.VolumeNames, manifest.HasVolumeData)
|
|
e.debugf("step 3 (user data) done in %v", time.Since(stepStart))
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 4: Create .fab bundle ---
|
|
job.setStep(step, "running", "")
|
|
stepStart = time.Now()
|
|
|
|
// v0.125.0 fail-loud guard (scenario B): never package a bundle whose manifest claims data
|
|
// that is not actually in the staging tree.
|
|
if err := assertBundleDataComplete(tmpDir, manifest); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("A csomag hiányos lenne — az export leállt: %v", err))
|
|
return
|
|
}
|
|
|
|
// Calculate total size
|
|
manifest.TotalSizeBytes = calcDirSize(tmpDir)
|
|
e.debugf("total bundle content size: %s (%d bytes)", humanizeBytes(manifest.TotalSizeBytes), manifest.TotalSizeBytes)
|
|
|
|
// Write manifest.json to tmpDir root
|
|
manifestData, err := manifest.Marshal()
|
|
if err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Manifest hiba: %v", err))
|
|
return
|
|
}
|
|
e.debugf("manifest JSON: %d bytes", len(manifestData))
|
|
if err := os.WriteFile(filepath.Join(tmpDir, "manifest.json"), manifestData, 0644); err != nil {
|
|
e.failJob(job, step, err.Error())
|
|
return
|
|
}
|
|
|
|
timestamp := time.Now().Format("20060102-150405")
|
|
fabName := fmt.Sprintf("%s_%s.fab", req.StackName, timestamp)
|
|
fabPath := filepath.Join(destDir, fabName)
|
|
e.debugf("target .fab path: %s", fabPath)
|
|
|
|
// Build tar.gz (to .tmp if encrypting, to final path if not)
|
|
targetPath := fabPath
|
|
if req.Password != "" {
|
|
targetPath = fabPath + ".tgz.tmp"
|
|
} else {
|
|
targetPath = fabPath + ".tmp"
|
|
}
|
|
|
|
e.debugf("creating tar.gz: %s", targetPath)
|
|
tgzStart := time.Now()
|
|
if err := createTarGz(targetPath, tmpDir); err != nil {
|
|
os.Remove(targetPath)
|
|
e.failJob(job, step, fmt.Sprintf("Csomag készítése sikertelen: %v", err))
|
|
return
|
|
}
|
|
if tgzInfo, err := os.Stat(targetPath); err == nil {
|
|
e.debugf("tar.gz created: %s (%s) in %v", targetPath, humanizeBytes(tgzInfo.Size()), time.Since(tgzStart))
|
|
}
|
|
|
|
job.setStep(step, "done", "")
|
|
step++
|
|
|
|
// --- Step 5: Encrypt (optional) ---
|
|
if req.Password != "" {
|
|
job.setStep(step, "running", "")
|
|
encStart := time.Now()
|
|
|
|
encPath := fabPath + ".tmp"
|
|
e.debugf("encrypting: %s → %s", targetPath, encPath)
|
|
if err := EncryptFile(targetPath, encPath, req.Password); err != nil {
|
|
os.Remove(targetPath)
|
|
e.failJob(job, step, fmt.Sprintf("Titkosítás sikertelen: %v", err))
|
|
return
|
|
}
|
|
os.Remove(targetPath)
|
|
targetPath = encPath
|
|
if encInfo, err := os.Stat(encPath); err == nil {
|
|
e.debugf("encryption done: %s in %v", humanizeBytes(encInfo.Size()), time.Since(encStart))
|
|
}
|
|
|
|
job.setStep(step, "done", "")
|
|
}
|
|
|
|
// Atomic rename to final path
|
|
e.debugf("atomic rename: %s → %s", targetPath, fabPath)
|
|
if err := os.Rename(targetPath, fabPath); err != nil {
|
|
e.failJob(job, step, fmt.Sprintf("Fájl átnevezés sikertelen: %v", err))
|
|
return
|
|
}
|
|
|
|
// Record result
|
|
stat, _ := os.Stat(fabPath)
|
|
job.mu.Lock()
|
|
job.OutputPath = fabPath
|
|
if stat != nil {
|
|
job.OutputSize = humanizeBytes(stat.Size())
|
|
}
|
|
job.mu.Unlock()
|
|
|
|
e.logger.Printf("[INFO] Export completed: %s → %s (%s) in %v", req.StackName, fabPath, job.OutputSize, time.Since(exportStart))
|
|
}
|
|
|
|
func (e *Exporter) failJob(job *Job, stepIdx int, msg string) {
|
|
job.setStep(stepIdx, "failed", msg)
|
|
job.mu.Lock()
|
|
job.Error = msg
|
|
job.mu.Unlock()
|
|
e.logger.Printf("[ERROR] Export/import failed at step %d: %s", stepIdx, msg)
|
|
e.debugf("FAIL at step %d: %s", stepIdx, msg)
|
|
}
|
|
|
|
// GetDebugInfo returns diagnostic information about the exporter state.
|
|
func (e *Exporter) GetDebugInfo() map[string]interface{} {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
|
|
info := map[string]interface{}{
|
|
"debug_enabled": e.debug,
|
|
"version": e.version,
|
|
"has_active_job": e.activeJob != nil,
|
|
}
|
|
|
|
if e.activeJob != nil {
|
|
info["active_job"] = e.activeJob.Snapshot()
|
|
}
|
|
|
|
return info
|
|
}
|
|
|
|
// copyStackConfig copies all relevant config files from the stack dir.
|
|
// app.yaml is saved with decrypted (plaintext) secrets for portability.
|
|
func copyStackConfig(stackDir, configDir, stackName string, provider ExportStackProvider) ([]string, error) {
|
|
var copied []string
|
|
|
|
entries, err := os.ReadDir(stackDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
// Skip temp files
|
|
if strings.HasSuffix(name, ".tmp") {
|
|
continue
|
|
}
|
|
|
|
src := filepath.Join(stackDir, name)
|
|
dst := filepath.Join(configDir, name)
|
|
|
|
// app.yaml: save with plaintext secrets
|
|
if name == "app.yaml" {
|
|
env := provider.GetDecryptedEnv(stackName)
|
|
if env != nil {
|
|
if err := writeDecryptedAppYaml(dst, env); err != nil {
|
|
return nil, fmt.Errorf("writing decrypted app.yaml: %w", err)
|
|
}
|
|
copied = append(copied, name)
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Copy file as-is
|
|
if err := copyFile(src, dst); err != nil {
|
|
return nil, fmt.Errorf("copying %s: %w", name, err)
|
|
}
|
|
copied = append(copied, name)
|
|
}
|
|
|
|
return copied, nil
|
|
}
|
|
|
|
// writeDecryptedAppYaml writes a plaintext app.yaml with the given env map.
|
|
func writeDecryptedAppYaml(dst string, env map[string]string) error {
|
|
var sb strings.Builder
|
|
sb.WriteString("# Exported by felhom-controller — plaintext secrets\n")
|
|
sb.WriteString("deployed: true\n")
|
|
sb.WriteString("env:\n")
|
|
for k, v := range env {
|
|
// YAML-safe: quote values
|
|
sb.WriteString(fmt.Sprintf(" %s: %q\n", k, v))
|
|
}
|
|
return os.WriteFile(dst, []byte(sb.String()), 0644)
|
|
}
|
|
|
|
// dumpDatabase discovers and dumps the database for a stack.
|
|
func (e *Exporter) dumpDatabase(stackName, dbDir string, manifest *Manifest) bool {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
e.debugf("discovering databases (looking for stack %s)...", stackName)
|
|
dbs, err := appbackup.DiscoverDatabases(ctx, e.logger, e.debug, nil)
|
|
if err != nil {
|
|
e.logger.Printf("[WARN] Export: DB discovery error: %v", err)
|
|
return false
|
|
}
|
|
e.debugf("found %d databases total", len(dbs))
|
|
for i := range dbs {
|
|
e.debugf(" db[%d]: stack=%s container=%s type=%s", i, dbs[i].StackName, dbs[i].ContainerName, dbs[i].DBType)
|
|
}
|
|
|
|
var stackDB *appbackup.DiscoveredDB
|
|
for i := range dbs {
|
|
if dbs[i].StackName == stackName {
|
|
stackDB = &dbs[i]
|
|
break
|
|
}
|
|
}
|
|
if stackDB == nil {
|
|
e.debugf("no database container found for stack %s", stackName)
|
|
return false
|
|
}
|
|
e.debugf("matched DB: container=%s type=%s", stackDB.ContainerName, stackDB.DBType)
|
|
|
|
dumpStart := time.Now()
|
|
result := appbackup.DumpOne(ctx, *stackDB, dbDir, e.logger, e.debug)
|
|
if result.Error != nil {
|
|
e.logger.Printf("[WARN] Export: DB dump failed for %s: %v", stackName, result.Error)
|
|
return false
|
|
}
|
|
e.debugf("DB dump completed in %v: %s", time.Since(dumpStart), result.FilePath)
|
|
|
|
// Gzip the dump
|
|
if result.FilePath != "" {
|
|
gzPath := result.FilePath + ".gz"
|
|
gzStart := time.Now()
|
|
if err := gzipFile(result.FilePath, gzPath); err != nil {
|
|
e.logger.Printf("[WARN] Export: gzip dump failed: %v", err)
|
|
} else {
|
|
if origInfo, _ := os.Stat(result.FilePath); origInfo != nil {
|
|
if gzInfo, _ := os.Stat(gzPath); gzInfo != nil {
|
|
e.debugf("gzip: %s → %s (ratio %.1f%%) in %v",
|
|
humanizeBytes(origInfo.Size()), humanizeBytes(gzInfo.Size()),
|
|
float64(gzInfo.Size())/float64(origInfo.Size())*100, time.Since(gzStart))
|
|
}
|
|
}
|
|
os.Remove(result.FilePath)
|
|
}
|
|
}
|
|
|
|
manifest.HasDatabase = true
|
|
manifest.DBType = string(stackDB.DBType)
|
|
|
|
return true
|
|
}
|
|
|
|
// exportHDDData copies HDD bind mount data for the export. v0.130.0 (C6B-F1 §8): a basename
|
|
// collision between two mounts is a FATAL error (the manifest keys tars by basename; the old
|
|
// code silently overwrote the first tar). A non-existent mount is still soft-skipped (honestly
|
|
// absent from the manifest — the anti-hollow guard catches total emptiness).
|
|
func (e *Exporter) exportHDDData(req ExportRequest, dataDir string, manifest *Manifest) error {
|
|
stackName := req.StackName
|
|
hddDir := filepath.Join(dataDir, "hdd")
|
|
os.MkdirAll(hddDir, 0755)
|
|
|
|
mounts := e.provider.GetStackHDDMounts(stackName)
|
|
e.debugf("HDD mounts for %s: %v (%d total)", stackName, mounts, len(mounts))
|
|
if len(mounts) == 0 {
|
|
e.debugf("no HDD mounts — skipping HDD data export")
|
|
return nil
|
|
}
|
|
|
|
// Task 4: the class-scoped plan. Legacy / no-block apps get an EMPTY plan (all mounts kept, root
|
|
// tar with zero excludes) → byte-identical v0.130.0 capture.
|
|
plan := e.computeFabPlan(req, mounts)
|
|
// R-203: a NAMESPACE ROOT, not the drive path (identical on an enrolled drive; one segment short
|
|
// on the system-data fallback).
|
|
ud := appbackup.UserdataDir(filepath.Clean(e.provider.GetStackNamespaceRoot(stackName)))
|
|
|
|
claimed := make(map[string]string) // subdir → mount that claimed it
|
|
for _, mount := range mounts {
|
|
if plan.SkipMounts[filepath.Clean(mount)] {
|
|
e.debugf("HDD mount %s skipped — not selected (class-scoped plan)", mount)
|
|
continue
|
|
}
|
|
isUserdataRoot := filepath.Clean(mount) == filepath.Clean(ud)
|
|
if isUserdataRoot && plan.SkipUserdataTar {
|
|
e.debugf("userdata root %s skipped — no selected userdata bind (Scenario B)", mount)
|
|
continue
|
|
}
|
|
if _, err := os.Stat(mount); os.IsNotExist(err) {
|
|
e.debugf("HDD mount %s does not exist — skipping", mount)
|
|
continue
|
|
}
|
|
subdir := filepath.Base(mount)
|
|
// C6B-F1 §8 (v0.130.0): the manifest keys HDD tars by BASENAME (the import side maps a
|
|
// basename back to a path), so two mounts sharing a basename cannot round-trip — the old
|
|
// code silently overwrote the first tar with the second (silent partial data loss).
|
|
// Renaming can't help either (the import couldn't map the new name), so the only honest
|
|
// outcome is a loud failure.
|
|
if prev, dup := claimed[subdir]; dup {
|
|
return util.MsgError("err.appexport.ket_adatkonyvtar_azonos_nevvel_vegzodik_es", subdir, prev, mount)
|
|
}
|
|
claimed[subdir] = mount
|
|
|
|
tarPath := filepath.Join(hddDir, subdir+".tar")
|
|
var excludes []string
|
|
if isUserdataRoot {
|
|
excludes = plan.UserdataExcludeRels // R1-C: exclude-scoped root tar (empty for legacy)
|
|
}
|
|
e.debugf("tarring HDD mount: %s → %s (%d exclude(s))", mount, tarPath, len(excludes))
|
|
tarStart := time.Now()
|
|
if err := tarDirectoryExcluding(mount, tarPath, excludes); err != nil {
|
|
// v0.125.0: claim the subdir ONLY on success — a claimed-but-absent tar would trip
|
|
// the packaging assertion; an honestly-skipped mount stays out of the manifest.
|
|
e.logger.Printf("[WARN] Export: failed to tar %s (excluded from the bundle): %v", mount, err)
|
|
os.Remove(tarPath)
|
|
} else {
|
|
manifest.HDDSubdirs = append(manifest.HDDSubdirs, subdir)
|
|
if info, _ := os.Stat(tarPath); info != nil {
|
|
e.debugf("HDD tar complete: %s (%s) in %v", subdir, humanizeBytes(info.Size()), time.Since(tarStart))
|
|
}
|
|
}
|
|
}
|
|
manifest.HasHDDData = len(manifest.HDDSubdirs) > 0
|
|
return nil
|
|
}
|
|
|
|
// dockerExec is the docker-CLI seam (v0.125.0): runs `docker args...` with optional
|
|
// stdin/stdout STREAMING and returns captured stderr (truncated). The volume legs stream tars
|
|
// over the docker API (docker cp) — NEVER via `docker run -v <controller-path>` host mounts,
|
|
// which the daemon resolves against the GUEST filesystem and silently strands the tar when the
|
|
// controller itself runs containerized (the v0.124.0 HIGH finding). Package var so unit tests
|
|
// inject a recorder (no docker on test boxes).
|
|
var dockerExec = func(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) (string, error) {
|
|
cmd := exec.CommandContext(ctx, "docker", args...)
|
|
cmd.Stdin = stdin
|
|
cmd.Stdout = stdout
|
|
var errBuf bytes.Buffer
|
|
cmd.Stderr = &errBuf
|
|
err := cmd.Run()
|
|
stderr := strings.TrimSpace(errBuf.String())
|
|
if len(stderr) > 500 {
|
|
stderr = stderr[:500] + "..."
|
|
}
|
|
return stderr, err
|
|
}
|
|
|
|
// withVolumeHelper creates a stopped helper container pinning volName at /vol, runs fn(cid),
|
|
// and ALWAYS force-removes the helper — including on fn failure (no leaked alpine containers).
|
|
func (e *Exporter) withVolumeHelper(volName string, fn func(cid string) error) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
var cidBuf bytes.Buffer
|
|
stderr, err := dockerExec(ctx, nil, &cidBuf, "create", "-v", volName+":/vol", "alpine", "true")
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("creating helper container for volume %s: %s — %w", volName, stderr, err)
|
|
}
|
|
cid := strings.TrimSpace(cidBuf.String())
|
|
defer func() {
|
|
rmCtx, rmCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer rmCancel()
|
|
if _, rmErr := dockerExec(rmCtx, nil, nil, "rm", "-f", cid); rmErr != nil {
|
|
e.logger.Printf("[WARN] appexport: helper container %s cleanup failed: %v", cid, rmErr)
|
|
}
|
|
}()
|
|
return fn(cid)
|
|
}
|
|
|
|
// exportVolumeTar streams one volume's content into tarPath via `docker cp <cid>:/vol/. -`
|
|
// (tar on stdout — zero shared paths; live-probed 2026-07-13: content, subdirs, symlinks,
|
|
// empty files and uid/gid all round-trip).
|
|
func (e *Exporter) exportVolumeTar(volName, tarPath string) error {
|
|
return e.withVolumeHelper(volName, func(cid string) error {
|
|
f, err := os.Create(tarPath)
|
|
if err != nil {
|
|
return fmt.Errorf("creating %s: %w", tarPath, err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
defer cancel()
|
|
stderr, err := dockerExec(ctx, nil, f, "cp", cid+":/vol/.", "-")
|
|
closeErr := f.Close()
|
|
if err != nil {
|
|
os.Remove(tarPath)
|
|
return fmt.Errorf("streaming volume %s: %s — %w", volName, stderr, err)
|
|
}
|
|
if closeErr != nil {
|
|
os.Remove(tarPath)
|
|
return fmt.Errorf("flushing %s: %w", tarPath, closeErr)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// exportVolumeData exports the app's Docker named volumes. v0.130.0 (C6B-F1): runs for EVERY
|
|
// app — needs_hdd apps hold state in named volumes too (sonarr_config = the whole app DB); the
|
|
// pre-fix else-branch silently dropped them. v0.125.0: a failed volume export is FATAL (export
|
|
// must never report success on a hollow bundle — the pre-fix WARN+continue is exactly how the
|
|
// data-loss bundles were born).
|
|
func (e *Exporter) exportVolumeData(stackName, dataDir string, manifest *Manifest) error {
|
|
volDir := filepath.Join(dataDir, "volumes")
|
|
os.MkdirAll(volDir, 0755)
|
|
|
|
volumes := e.provider.GetDockerVolumes(stackName)
|
|
e.debugf("Docker volumes for %s: %v (%d total)", stackName, volumes, len(volumes))
|
|
if len(volumes) == 0 {
|
|
e.debugf("no Docker volumes — skipping volume data export")
|
|
return nil
|
|
}
|
|
|
|
for _, volName := range volumes {
|
|
tarPath := filepath.Join(volDir, volName+".tar")
|
|
e.debugf("exporting volume %s via docker cp streaming...", volName)
|
|
volStart := time.Now()
|
|
if err := e.exportVolumeTar(volName, tarPath); err != nil {
|
|
return fmt.Errorf("volume %s export failed: %w", volName, err)
|
|
}
|
|
if info, _ := os.Stat(tarPath); info != nil {
|
|
e.debugf("volume %s exported: %s in %v", volName, humanizeBytes(info.Size()), time.Since(volStart))
|
|
}
|
|
manifest.VolumeNames = append(manifest.VolumeNames, volName)
|
|
}
|
|
manifest.HasVolumeData = len(manifest.VolumeNames) > 0
|
|
return nil
|
|
}
|
|
|
|
// assertBundleDataComplete is the fail-loud post-export guard (v0.125.0, scenario B): every
|
|
// manifest-CLAIMED data tar must exist non-empty in the staging tree before packaging. A
|
|
// mismatch aborts the export — yesterday's outcome ("success" with a hollow bundle) is the
|
|
// one this exists to make impossible. v0.130.0 (C6B-F1 cause 3): also refuses a needs_hdd
|
|
// bundle that claims NO data at all — the claimed-tar checks pass trivially on 0 claims, which
|
|
// is how a discovery gap shipped hollow bundles right past the v0.125.0 net.
|
|
func assertBundleDataComplete(tmpDir string, manifest *Manifest) error {
|
|
for _, v := range manifest.VolumeNames {
|
|
fi, err := os.Stat(filepath.Join(tmpDir, "data", "volumes", v+".tar"))
|
|
if err != nil || fi.Size() == 0 {
|
|
return fmt.Errorf("bundle assertion: volume %q is claimed by the manifest but its tar is missing or empty", v)
|
|
}
|
|
}
|
|
for _, s := range manifest.HDDSubdirs {
|
|
fi, err := os.Stat(filepath.Join(tmpDir, "data", "hdd", s+".tar"))
|
|
if err != nil || fi.Size() == 0 {
|
|
return fmt.Errorf("bundle assertion: HDD subdir %q is claimed by the manifest but its tar is missing or empty", s)
|
|
}
|
|
}
|
|
// C6B-F1 cause 3 (v0.130.0): the claimed-tar checks above pass TRIVIALLY when discovery finds
|
|
// nothing (0 claims → 0 checks) — exactly how a 4.17 GB app shipped as a 2308-byte config-only
|
|
// bundle. A needs_hdd app with NO data of any kind is a hollow bundle by definition; refuse it
|
|
// loudly so a future discovery gap can never again ship silently.
|
|
if manifest.NeedsHDD && !manifest.HasHDDData && !manifest.HasVolumeData {
|
|
return util.MsgError("err.appexport.a_mentes_nem_tartalmaz_alkalmazasadatot_0")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// createTarGz creates a gzipped tar archive of a directory.
|
|
func createTarGz(outputPath, sourceDir string) error {
|
|
outFile, err := os.Create(outputPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer outFile.Close()
|
|
|
|
gw := gzip.NewWriter(outFile)
|
|
defer gw.Close()
|
|
|
|
tw := tar.NewWriter(gw)
|
|
defer tw.Close()
|
|
|
|
return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Get path relative to sourceDir
|
|
relPath, err := filepath.Rel(sourceDir, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if relPath == "." {
|
|
return nil
|
|
}
|
|
|
|
header, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header.Name = relPath
|
|
|
|
if err := tw.WriteHeader(header); err != nil {
|
|
return err
|
|
}
|
|
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
_, err = io.Copy(tw, f)
|
|
return err
|
|
})
|
|
}
|
|
|
|
// tarDirectory creates a tar (not gzipped) of a directory's contents. Thin wrapper over
|
|
// tarDirectoryExcluding (Task 4) with no excludes — its existing callers are unchanged.
|
|
func tarDirectory(sourceDir, outputPath string) error {
|
|
return tarDirectoryExcluding(sourceDir, outputPath, nil)
|
|
}
|
|
|
|
// gzipFile compresses a file with gzip.
|
|
func gzipFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
gw := gzip.NewWriter(out)
|
|
defer gw.Close()
|
|
|
|
_, err = io.Copy(gw, in)
|
|
return err
|
|
}
|
|
|
|
// copyFile copies a file from src to dst.
|
|
func copyFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
if _, err := io.Copy(out, in); err != nil {
|
|
return err
|
|
}
|
|
return out.Sync()
|
|
}
|
|
|
|
// calcDirSize recursively calculates total file size in a directory.
|
|
func calcDirSize(dir string) int64 {
|
|
var total int64
|
|
filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
|
|
if err != nil || info.IsDir() {
|
|
return nil
|
|
}
|
|
total += info.Size()
|
|
return nil
|
|
})
|
|
return total
|
|
}
|