controller: .fab volume legs stream via docker cp (fixes the containerized path-strand data loss, IA finding 1 HIGH) — zero shared paths both deployment shapes; export fails LOUD on any missing/empty claimed tar (no hollow bundles); import validates BEFORE it destroys (hollow bundle → app untouched); docker_run_volume_path_gate.py extinguishes the class (Tier-1/2 mounts documented host-visible)

Claude-Session: https://claude.ai/code/session_01GzammAMzsJTgpQHqxwM2bC
2026-07-13 10:42:02 +02:00
parent 811a0ef7a7
commit 466f42708e
5 changed files with 489 additions and 46 deletions
@@ -2,6 +2,7 @@ package appexport

 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"context"
 	"fmt"
@@ -324,7 +325,10 @@ func (e *Exporter) executeExport(req ExportRequest, job *Job) {
 		e.debugf("HDD data exported: subdirs=%v hasData=%v", manifest.HDDSubdirs, manifest.HasHDDData)
 	} else {
 		e.debugf("exporting Docker volumes for %s", req.StackName)
-		e.exportVolumeData(req.StackName, dataDir, manifest)
+		if err := e.exportVolumeData(req.StackName, dataDir, manifest); err != nil {
+			e.failJob(job, step, fmt.Sprintf("Kötet mentése sikertelen: %v", err))
+			return
+		}
 		e.debugf("volume data exported: volumes=%v hasData=%v", manifest.VolumeNames, manifest.HasVolumeData)
 	}
 	e.debugf("step 3 (user data) done in %v", time.Since(stepStart))
@@ -336,6 +340,13 @@ func (e *Exporter) executeExport(req ExportRequest, job *Job) {
 	job.setStep(step, "running", "")
 	stepStart = time.Now()

+	// v0.125.0 fail-loud guard (scenario B): never package a bundle whose manifest claims data
+	// that is not actually in the staging tree.
+	if err := assertBundleDataComplete(tmpDir, manifest); err != nil {
+		e.failJob(job, step, fmt.Sprintf("A csomag hiányos lenne — az export leállt: %v", err))
+		return
+	}
+
 	// Calculate total size
 	manifest.TotalSizeBytes = calcDirSize(tmpDir)
 	e.debugf("total bundle content size: %s (%d bytes)", humanizeBytes(manifest.TotalSizeBytes), manifest.TotalSizeBytes)
@@ -583,14 +594,17 @@ func (e *Exporter) exportHDDData(stackName, dataDir string, manifest *Manifest)
 			continue
 		}
 		subdir := filepath.Base(mount)
-		manifest.HDDSubdirs = append(manifest.HDDSubdirs, subdir)

 		tarPath := filepath.Join(hddDir, subdir+".tar")
 		e.debugf("tarring HDD mount: %s → %s", mount, tarPath)
 		tarStart := time.Now()
 		if err := tarDirectory(mount, tarPath); err != nil {
-			e.logger.Printf("[WARN] Export: failed to tar %s: %v", mount, err)
+			// v0.125.0: claim the subdir ONLY on success — a claimed-but-absent tar would trip
+			// the packaging assertion; an honestly-skipped mount stays out of the manifest.
+			e.logger.Printf("[WARN] Export: failed to tar %s (excluded from the bundle): %v", mount, err)
+			os.Remove(tarPath)
 		} else {
+			manifest.HDDSubdirs = append(manifest.HDDSubdirs, subdir)
 			if info, _ := os.Stat(tarPath); info != nil {
 				e.debugf("HDD tar complete: %s (%s) in %v", subdir, humanizeBytes(info.Size()), time.Since(tarStart))
 			}
@@ -599,8 +613,76 @@ func (e *Exporter) exportHDDData(stackName, dataDir string, manifest *Manifest)
 	manifest.HasHDDData = len(manifest.HDDSubdirs) > 0
 }

-// exportVolumeData exports Docker named volumes for apps without HDD storage.
-func (e *Exporter) exportVolumeData(stackName, dataDir string, manifest *Manifest) {
+// dockerExec is the docker-CLI seam (v0.125.0): runs `docker args...` with optional
+// stdin/stdout STREAMING and returns captured stderr (truncated). The volume legs stream tars
+// over the docker API (docker cp) — NEVER via `docker run -v <controller-path>` host mounts,
+// which the daemon resolves against the GUEST filesystem and silently strands the tar when the
+// controller itself runs containerized (the v0.124.0 HIGH finding). Package var so unit tests
+// inject a recorder (no docker on test boxes).
+var dockerExec = func(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Stdin = stdin
+	cmd.Stdout = stdout
+	var errBuf bytes.Buffer
+	cmd.Stderr = &errBuf
+	err := cmd.Run()
+	stderr := strings.TrimSpace(errBuf.String())
+	if len(stderr) > 500 {
+		stderr = stderr[:500] + "..."
+	}
+	return stderr, err
+}
+
+// withVolumeHelper creates a stopped helper container pinning volName at /vol, runs fn(cid),
+// and ALWAYS force-removes the helper — including on fn failure (no leaked alpine containers).
+func (e *Exporter) withVolumeHelper(volName string, fn func(cid string) error) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	var cidBuf bytes.Buffer
+	stderr, err := dockerExec(ctx, nil, &cidBuf, "create", "-v", volName+":/vol", "alpine", "true")
+	cancel()
+	if err != nil {
+		return fmt.Errorf("creating helper container for volume %s: %s — %w", volName, stderr, err)
+	}
+	cid := strings.TrimSpace(cidBuf.String())
+	defer func() {
+		rmCtx, rmCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer rmCancel()
+		if _, rmErr := dockerExec(rmCtx, nil, nil, "rm", "-f", cid); rmErr != nil {
+			e.logger.Printf("[WARN] appexport: helper container %s cleanup failed: %v", cid, rmErr)
+		}
+	}()
+	return fn(cid)
+}
+
+// exportVolumeTar streams one volume's content into tarPath via `docker cp <cid>:/vol/. -`
+// (tar on stdout — zero shared paths; live-probed 2026-07-13: content, subdirs, symlinks,
+// empty files and uid/gid all round-trip).
+func (e *Exporter) exportVolumeTar(volName, tarPath string) error {
+	return e.withVolumeHelper(volName, func(cid string) error {
+		f, err := os.Create(tarPath)
+		if err != nil {
+			return fmt.Errorf("creating %s: %w", tarPath, err)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		stderr, err := dockerExec(ctx, nil, f, "cp", cid+":/vol/.", "-")
+		closeErr := f.Close()
+		if err != nil {
+			os.Remove(tarPath)
+			return fmt.Errorf("streaming volume %s: %s — %w", volName, stderr, err)
+		}
+		if closeErr != nil {
+			os.Remove(tarPath)
+			return fmt.Errorf("flushing %s: %w", tarPath, closeErr)
+		}
+		return nil
+	})
+}
+
+// exportVolumeData exports Docker named volumes for apps without HDD storage. v0.125.0: a
+// failed volume export is FATAL (export must never report success on a hollow bundle — the
+// pre-fix WARN+continue is exactly how the data-loss bundles were born).
+func (e *Exporter) exportVolumeData(stackName, dataDir string, manifest *Manifest) error {
 	volDir := filepath.Join(dataDir, "volumes")
 	os.MkdirAll(volDir, 0755)

@@ -608,28 +690,15 @@ func (e *Exporter) exportVolumeData(stackName, dataDir string, manifest *Manifes
 	e.debugf("Docker volumes for %s: %v (%d total)", stackName, volumes, len(volumes))
 	if len(volumes) == 0 {
 		e.debugf("no Docker volumes — skipping volume data export")
-		return
+		return nil
 	}

 	for _, volName := range volumes {
 		tarPath := filepath.Join(volDir, volName+".tar")
-		e.debugf("exporting volume %s via docker run alpine tar...", volName)
+		e.debugf("exporting volume %s via docker cp streaming...", volName)
 		volStart := time.Now()
-
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-		cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
-			"-v", volName+":/vol:ro",
-			"-v", volDir+":/out",
-			"alpine", "tar", "cf", "/out/"+volName+".tar", "-C", "/vol", ".")
-		out, err := cmd.CombinedOutput()
-		cancel()
-
-		if err != nil {
-			e.logger.Printf("[WARN] Export: volume %s export failed: %s — %v",
-				volName, strings.TrimSpace(string(out)), err)
-			e.debugf("volume %s export failed: %s", volName, strings.TrimSpace(string(out)))
-			os.Remove(tarPath)
-			continue
+		if err := e.exportVolumeTar(volName, tarPath); err != nil {
+			return fmt.Errorf("volume %s export failed: %w", volName, err)
 		}
 		if info, _ := os.Stat(tarPath); info != nil {
 			e.debugf("volume %s exported: %s in %v", volName, humanizeBytes(info.Size()), time.Since(volStart))
@@ -637,6 +706,27 @@ func (e *Exporter) exportVolumeData(stackName, dataDir string, manifest *Manifes
 		manifest.VolumeNames = append(manifest.VolumeNames, volName)
 	}
 	manifest.HasVolumeData = len(manifest.VolumeNames) > 0
+	return nil
+}
+
+// assertBundleDataComplete is the fail-loud post-export guard (v0.125.0, scenario B): every
+// manifest-CLAIMED data tar must exist non-empty in the staging tree before packaging. A
+// mismatch aborts the export — yesterday's outcome ("success" with a hollow bundle) is the
+// one this exists to make impossible.
+func assertBundleDataComplete(tmpDir string, manifest *Manifest) error {
+	for _, v := range manifest.VolumeNames {
+		fi, err := os.Stat(filepath.Join(tmpDir, "data", "volumes", v+".tar"))
+		if err != nil || fi.Size() == 0 {
+			return fmt.Errorf("bundle assertion: volume %q is claimed by the manifest but its tar is missing or empty", v)
+		}
+	}
+	for _, s := range manifest.HDDSubdirs {
+		fi, err := os.Stat(filepath.Join(tmpDir, "data", "hdd", s+".tar"))
+		if err != nil || fi.Size() == 0 {
+			return fmt.Errorf("bundle assertion: HDD subdir %q is claimed by the manifest but its tar is missing or empty", s)
+		}
+	}
+	return nil
 }

 // createTarGz creates a gzipped tar archive of a directory.