R-1214: the second-drive copy runs in a 512 MB memory-capped container for /mnt paths (its page cache no longer evicts the apps) — measured on 9202; guarded against a path the daemon sees differently; unreleased

CHANGELOG.md

@@ -1,3 +1,19 @@
+## Unreleased — the second-drive copy no longer squeezes the apps' memory (R-1214), 2026-10-22 night
+
+**MinAgent: 0.158.1** (unchanged). Committed, not released.
+
+- **R-1214.** The night soak's one missed pass line (memory „full" pressure above 20 % for ~3 min in the 03:30
+  second-drive job on the 8 GB box). Measured on 9202 limited to 4 GB / 4 GB swap with 3.5 GB in use and an active
+  1.8 GB working set (felhom.eu `audits/night-2026-10-22/P4-R1214/`), a 6 GB rsync: in the controller's process (as
+  the copy runs today) 6.9–16.8 % „full" (10-s average) and 2.5–5.7 s of full stall; the same rsync in a container
+  capped at 512 MB 0.9–1.0 % and 0.3–0.4 s. The cause: the copy's page cache is charged to the guest and evicts the
+  apps' working memory. `rsyncMirror` now runs rsync in a `docker run --rm --memory 512m --memory-swap 512m --network
+  none` container of the controller's own image (it carries rsync), source read-only — ONLY when both paths are under
+  `/mnt/`, where the controller and the Docker daemon see the same paths: a path on the controller's own data volume
+  differs for the daemon, and a capped `--delete` there would mirror an EMPTY folder over the copy. Those paths, a box
+  whose image cannot be read and a container that cannot start (exit 125) keep the plain in-process rsync; an rsync
+  failure inside the container is an error. Tests `TestR1214_*` (3, Docker-free seams), red-proved both ways.
+
 ## Unreleased — two data-loss candidates fixed (R-1187 c, R-1189 d), 2026-10-22 night
 
 **MinAgent: 0.158.1** (unchanged). Committed, not released (night brief 2026-10-22: no controller release tonight).

controller/internal/backup/r1214_capped_copy_test.go

@@ -0,0 +1,93 @@
+package backup
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// R-1214: the second-drive copy runs in a memory-capped container ONLY for paths the controller and the Docker daemon
+// see identically (/mnt/…); every other path, a box with no readable image, and a container that cannot start keep the
+// plain in-process rsync; an rsync failure inside the container is an error, never a silent second copy.
+// RED-PROOF: with the tier2CappedPath guard removed, TestR1214_APathOutsideMntNeverGoesToDocker fails (the copy of a
+// controller-volume path would go to the container, where it is an EMPTY folder and `--delete` empties the copy);
+// with the capped branch removed, TestR1214_MntPathsAreCopiedCapped fails — both seen failing 2026-10-22.
+type cappedRec struct{ calls [][]string }
+
+func withSeams(t *testing.T, root string, img string, run func(args []string) ([]byte, int, error)) *cappedRec {
+	t.Helper()
+	rec := &cappedRec{}
+	oldRoot, oldImg, oldRun, oldLog := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf
+	tier2CappedRoot = root
+	tier2SelfImage = func() string { return img }
+	tier2RunCapped = func(_ context.Context, args []string) ([]byte, int, error) {
+		rec.calls = append(rec.calls, args)
+		return run(args)
+	}
+	tier2Logf = func(string, ...any) {}
+	t.Cleanup(func() { tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf = oldRoot, oldImg, oldRun, oldLog })
+	return rec
+}
+
+func TestR1214_MntPathsAreCopiedCapped(t *testing.T) {
+	root := t.TempDir() + "/"
+	src, dst := filepath.Join(root, "drive1", "unit"), filepath.Join(root, "drive2", "unit")
+	os.MkdirAll(src, 0755)
+	rec := withSeams(t, root, "gitea.dooplex.hu/admin/felhom-controller:9.9.9", func([]string) ([]byte, int, error) { return nil, 0, nil })
+	if err := rsyncMirror(src, dst); err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.calls) != 1 {
+		t.Fatalf("docker calls = %d, want 1", len(rec.calls))
+	}
+	got := strings.Join(rec.calls[0], " ")
+	for _, want := range []string{"run --rm", "--memory 512m", "--memory-swap 512m", "--network none",
+		"-v " + src + ":" + src + ":ro", "-v " + dst + ":" + dst + " ", "--entrypoint rsync gitea.dooplex.hu/admin/felhom-controller:9.9.9 -a --delete " + src + "/ " + dst + "/"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("the capped copy lacks %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestR1214_APathOutsideMntNeverGoesToDocker(t *testing.T) {
+	root := t.TempDir() + "/"
+	other := t.TempDir() // a path the daemon would see differently (the controller's own data volume)
+	src, dst := filepath.Join(other, "unit"), filepath.Join(root, "drive2", "unit")
+	os.MkdirAll(src, 0755)
+	os.WriteFile(filepath.Join(src, "photo.jpg"), []byte("x"), 0644)
+	rec := withSeams(t, root, "img", func([]string) ([]byte, int, error) { return nil, 0, nil })
+	if err := rsyncMirror(src, dst); err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.calls) != 0 {
+		t.Fatalf("a path outside %s went to the container: %v", root, rec.calls)
+	}
+	if _, err := os.Stat(filepath.Join(dst, "photo.jpg")); err != nil {
+		t.Fatal("the plain copy did not run")
+	}
+}
+
+func TestR1214_ANotStartedContainerFallsBackAnRsyncErrorDoesNot(t *testing.T) {
+	root := t.TempDir() + "/"
+	src, dst := filepath.Join(root, "a", "unit"), filepath.Join(root, "b", "unit")
+	os.MkdirAll(src, 0755)
+	os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0644)
+	withSeams(t, root, "img", func([]string) ([]byte, int, error) { return []byte("no such image"), 125, errors.New("exit 125") })
+	if err := rsyncMirror(src, dst); err != nil {
+		t.Fatalf("a container that cannot start must fall back to the plain copy: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dst, "f")); err != nil {
+		t.Fatal("the fallback copy did not run")
+	}
+	withSeams(t, root, "img", func([]string) ([]byte, int, error) { return []byte("rsync error"), 23, errors.New("exit 23") })
+	if err := rsyncMirror(src, dst); err == nil {
+		t.Fatal("an rsync failure inside the container must be an error")
+	}
+	withSeams(t, root, "", func([]string) ([]byte, int, error) { t.Fatal("no image → no container"); return nil, 0, nil })
+	if err := rsyncMirror(src, dst); err != nil {
+		t.Fatalf("no readable image → the plain copy: %v", err)
+	}
+}

controller/internal/backup/tier2.go

@@ -5,12 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
 )
@@ -770,6 +772,16 @@ func (m *Manager) tier2NoTargetReason(err error) string {
 // --- helpers ---
 
 // rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning).
+//
+// R-1214 (night soak 2026-10-21; measured on 9202 2026-10-22, felhom.eu `audits/night-2026-10-22/P4-R1214/`): run
+// in the controller's own process, the copy's page cache is charged to the GUEST's memory and evicts the apps' working
+// memory — on a 4 GB guest with 3.5 GB in use a 6 GB copy drove memory „full" pressure to 6.9–16.8 % (10-s average,
+// 2.5–5.7 s of full stall); the same copy inside a container capped at 512 MB: 0.9–1.0 %, 0.3–0.4 s. So the copy runs
+// in a capped container of the controller's own image (it carries rsync) — ONLY when both paths are under /mnt/,
+// where the controller and the Docker daemon see the same paths. Any other path (the controller's own data volume)
+// differs for the daemon: a capped `--delete` there would mirror an EMPTY folder over the copy. Those paths, a box
+// whose image cannot be read, and a container that cannot START (docker's own exit 125) take the plain rsync, as
+// before. Pinned by TestR1214_*.
 func rsyncMirror(src, dst string) error {
 	if err := os.MkdirAll(dst, 0755); err != nil {
 		return fmt.Errorf("mkdir %s: %w", dst, err)
@@ -777,7 +789,23 @@ func rsyncMirror(src, dst string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
 	defer cancel()
 	// Trailing slashes: copy the CONTENTS of src into dst.
-	cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/")
+	s, d := strings.TrimRight(src, "/"), strings.TrimRight(dst, "/")
+	rs := []string{"-a", "--delete", s + "/", d + "/"}
+	if tier2CappedPath(s) && tier2CappedPath(d) {
+		if img := tier2SelfImage(); img != "" {
+			args := append([]string{"run", "--rm", "--memory", tier2CopyMemCap, "--memory-swap", tier2CopyMemCap,
+				"--network", "none", "-v", s + ":" + s + ":ro", "-v", d + ":" + d, "--entrypoint", "rsync", img}, rs...)
+			out, code, err := tier2RunCapped(ctx, args)
+			if err == nil {
+				return nil
+			}
+			if code != 125 {
+				return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
+			}
+			tier2Logf("[WARN] [tier2] the capped copy container did not start (%s) — copying in-process, as before (R-1214)", strings.TrimSpace(string(out)))
+		}
+	}
+	cmd := exec.CommandContext(ctx, "rsync", rs...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
@@ -785,6 +813,39 @@ func rsyncMirror(src, dst string) error {
 	return nil
 }
 
+// tier2CopyMemCap bounds the capped copy's memory: rsync itself needs a few MB per 100 000 files; the rest is the page
+// cache the copy creates, reclaimed inside the cap instead of from the apps.
+const tier2CopyMemCap = "512m"
+
+// tier2CappedPath: a path the controller and the Docker daemon see identically (the guest's /mnt is bind-mounted into
+// the controller at /mnt). Never a relative path, never `..`.
+func tier2CappedPath(p string) bool {
+	return strings.HasPrefix(p, tier2CappedRoot) && filepath.Clean(p) == p
+}
+
+// tier2CappedRoot is "/mnt/" in production (a test seam: tests point it at a temp dir).
+var tier2CappedRoot = "/mnt/"
+
+// Seams (tests never reach Docker — tests that do act on the host they run on).
+var (
+	tier2SelfImage = func() string {
+		out, err := dockerexec.Command("docker", "inspect", "felhom-controller", "--format", "{{.Config.Image}}").Output()
+		if err != nil {
+			return ""
+		}
+		return strings.TrimSpace(string(out))
+	}
+	tier2RunCapped = func(ctx context.Context, args []string) ([]byte, int, error) {
+		out, err := dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput()
+		var ee *exec.ExitError
+		if errors.As(err, &ee) {
+			return out, ee.ExitCode(), err
+		}
+		return out, 0, err
+	}
+	tier2Logf = log.Printf
+)
+
 // tier2UnitPreservedWarning is the customer-facing half of the R-403 skip. It rides in
 // CrossDriveBackup.LastWarning, which the per-app card already renders, so the refusal reaches the
 // SURFACE and not only the log — the shape recordTier2NoTarget established.

controller/scripts/docker_run_volume_path_gate.py

@@ -14,6 +14,12 @@ ROOTS = ["internal", "cmd"]
 
 # (file suffix, substring that must appear on the "-v" line, why it is safe)
 ALLOWLIST = [
+    ("internal/backup/tier2.go", '"-v", s + ":" + s + ":ro", "-v", d + ":" + d',
+     "R-1214 (2026-10-22): the second-drive copy's memory-capped rsync container. Reached ONLY when both paths pass "
+     "tier2CappedPath — under /mnt/, which the golden deployment bind-mounts into the controller at the IDENTICAL "
+     "path (container-inspect on 9202 2026-10-22: /mnt -> /mnt) — so the daemon resolves them correctly; every other "
+     "path (the controller's own data volume) takes the in-process rsync. Pinned by "
+     "TestR1214_APathOutsideMntNeverGoesToDocker"),
     ("internal/appbackup/dbdump.go", '"psql", "-v"',
      "psql's own -v flag (ON_ERROR_STOP) — not a docker mount at all"),
     ("internal/appexport/estimate.go", '"-v", volumeName+":/vol:ro"',