From 03c0a693b8fce1d93b26e0cce278044a228e26a6 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 22 Oct 2026 17:45:47 +0200 Subject: [PATCH] =?UTF-8?q?R-1214:=20the=20second-drive=20copy=20runs=20in?= =?UTF-8?q?=20a=20512=20MB=20memory-capped=20container=20for=20/mnt=20path?= =?UTF-8?q?s=20(its=20page=20cache=20no=20longer=20evicts=20the=20apps)=20?= =?UTF-8?q?=E2=80=94=20measured=20on=209202;=20guarded=20against=20a=20pat?= =?UTF-8?q?h=20the=20daemon=20sees=20differently;=20unreleased?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 16 ++++ .../internal/backup/r1214_capped_copy_test.go | 93 +++++++++++++++++++ controller/internal/backup/tier2.go | 63 ++++++++++++- .../scripts/docker_run_volume_path_gate.py | 6 ++ 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 controller/internal/backup/r1214_capped_copy_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index abcd997..e45f66f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## Unreleased — the second-drive copy no longer squeezes the apps' memory (R-1214), 2026-10-22 night + +**MinAgent: 0.158.1** (unchanged). Committed, not released. + +- **R-1214.** The night soak's one missed pass line (memory „full" pressure above 20 % for ~3 min in the 03:30 + second-drive job on the 8 GB box). Measured on 9202 limited to 4 GB / 4 GB swap with 3.5 GB in use and an active + 1.8 GB working set (felhom.eu `audits/night-2026-10-22/P4-R1214/`), a 6 GB rsync: in the controller's process (as + the copy runs today) 6.9–16.8 % „full" (10-s average) and 2.5–5.7 s of full stall; the same rsync in a container + capped at 512 MB 0.9–1.0 % and 0.3–0.4 s. The cause: the copy's page cache is charged to the guest and evicts the + apps' working memory. `rsyncMirror` now runs rsync in a `docker run --rm --memory 512m --memory-swap 512m --network + none` container of the controller's own image (it carries rsync), source read-only — ONLY when both paths are under + `/mnt/`, where the controller and the Docker daemon see the same paths: a path on the controller's own data volume + differs for the daemon, and a capped `--delete` there would mirror an EMPTY folder over the copy. Those paths, a box + whose image cannot be read and a container that cannot start (exit 125) keep the plain in-process rsync; an rsync + failure inside the container is an error. Tests `TestR1214_*` (3, Docker-free seams), red-proved both ways. + ## Unreleased — two data-loss candidates fixed (R-1187 c, R-1189 d), 2026-10-22 night **MinAgent: 0.158.1** (unchanged). Committed, not released (night brief 2026-10-22: no controller release tonight). diff --git a/controller/internal/backup/r1214_capped_copy_test.go b/controller/internal/backup/r1214_capped_copy_test.go new file mode 100644 index 0000000..296434c --- /dev/null +++ b/controller/internal/backup/r1214_capped_copy_test.go @@ -0,0 +1,93 @@ +package backup + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// R-1214: the second-drive copy runs in a memory-capped container ONLY for paths the controller and the Docker daemon +// see identically (/mnt/…); every other path, a box with no readable image, and a container that cannot start keep the +// plain in-process rsync; an rsync failure inside the container is an error, never a silent second copy. +// RED-PROOF: with the tier2CappedPath guard removed, TestR1214_APathOutsideMntNeverGoesToDocker fails (the copy of a +// controller-volume path would go to the container, where it is an EMPTY folder and `--delete` empties the copy); +// with the capped branch removed, TestR1214_MntPathsAreCopiedCapped fails — both seen failing 2026-10-22. +type cappedRec struct{ calls [][]string } + +func withSeams(t *testing.T, root string, img string, run func(args []string) ([]byte, int, error)) *cappedRec { + t.Helper() + rec := &cappedRec{} + oldRoot, oldImg, oldRun, oldLog := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf + tier2CappedRoot = root + tier2SelfImage = func() string { return img } + tier2RunCapped = func(_ context.Context, args []string) ([]byte, int, error) { + rec.calls = append(rec.calls, args) + return run(args) + } + tier2Logf = func(string, ...any) {} + t.Cleanup(func() { tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf = oldRoot, oldImg, oldRun, oldLog }) + return rec +} + +func TestR1214_MntPathsAreCopiedCapped(t *testing.T) { + root := t.TempDir() + "/" + src, dst := filepath.Join(root, "drive1", "unit"), filepath.Join(root, "drive2", "unit") + os.MkdirAll(src, 0755) + rec := withSeams(t, root, "gitea.dooplex.hu/admin/felhom-controller:9.9.9", func([]string) ([]byte, int, error) { return nil, 0, nil }) + if err := rsyncMirror(src, dst); err != nil { + t.Fatal(err) + } + if len(rec.calls) != 1 { + t.Fatalf("docker calls = %d, want 1", len(rec.calls)) + } + got := strings.Join(rec.calls[0], " ") + for _, want := range []string{"run --rm", "--memory 512m", "--memory-swap 512m", "--network none", + "-v " + src + ":" + src + ":ro", "-v " + dst + ":" + dst + " ", "--entrypoint rsync gitea.dooplex.hu/admin/felhom-controller:9.9.9 -a --delete " + src + "/ " + dst + "/"} { + if !strings.Contains(got, want) { + t.Fatalf("the capped copy lacks %q:\n%s", want, got) + } + } +} + +func TestR1214_APathOutsideMntNeverGoesToDocker(t *testing.T) { + root := t.TempDir() + "/" + other := t.TempDir() // a path the daemon would see differently (the controller's own data volume) + src, dst := filepath.Join(other, "unit"), filepath.Join(root, "drive2", "unit") + os.MkdirAll(src, 0755) + os.WriteFile(filepath.Join(src, "photo.jpg"), []byte("x"), 0644) + rec := withSeams(t, root, "img", func([]string) ([]byte, int, error) { return nil, 0, nil }) + if err := rsyncMirror(src, dst); err != nil { + t.Fatal(err) + } + if len(rec.calls) != 0 { + t.Fatalf("a path outside %s went to the container: %v", root, rec.calls) + } + if _, err := os.Stat(filepath.Join(dst, "photo.jpg")); err != nil { + t.Fatal("the plain copy did not run") + } +} + +func TestR1214_ANotStartedContainerFallsBackAnRsyncErrorDoesNot(t *testing.T) { + root := t.TempDir() + "/" + src, dst := filepath.Join(root, "a", "unit"), filepath.Join(root, "b", "unit") + os.MkdirAll(src, 0755) + os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0644) + withSeams(t, root, "img", func([]string) ([]byte, int, error) { return []byte("no such image"), 125, errors.New("exit 125") }) + if err := rsyncMirror(src, dst); err != nil { + t.Fatalf("a container that cannot start must fall back to the plain copy: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, "f")); err != nil { + t.Fatal("the fallback copy did not run") + } + withSeams(t, root, "img", func([]string) ([]byte, int, error) { return []byte("rsync error"), 23, errors.New("exit 23") }) + if err := rsyncMirror(src, dst); err == nil { + t.Fatal("an rsync failure inside the container must be an error") + } + withSeams(t, root, "", func([]string) ([]byte, int, error) { t.Fatal("no image → no container"); return nil, 0, nil }) + if err := rsyncMirror(src, dst); err != nil { + t.Fatalf("no readable image → the plain copy: %v", err) + } +} diff --git a/controller/internal/backup/tier2.go b/controller/internal/backup/tier2.go index 5674e5e..66160cd 100644 --- a/controller/internal/backup/tier2.go +++ b/controller/internal/backup/tier2.go @@ -5,12 +5,14 @@ import ( "errors" "fmt" "gitea.dooplex.hu/admin/felhom-controller/internal/util" + "log" "os" "os/exec" "path/filepath" "strings" "time" + "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" "gitea.dooplex.hu/admin/felhom-controller/internal/system" ) @@ -770,6 +772,16 @@ func (m *Manager) tier2NoTargetReason(err error) string { // --- helpers --- // rsyncMirror mirrors src→dst with rsync -a --delete (exact copy, browsable on disk, no versioning). +// +// R-1214 (night soak 2026-10-21; measured on 9202 2026-10-22, felhom.eu `audits/night-2026-10-22/P4-R1214/`): run +// in the controller's own process, the copy's page cache is charged to the GUEST's memory and evicts the apps' working +// memory — on a 4 GB guest with 3.5 GB in use a 6 GB copy drove memory „full" pressure to 6.9–16.8 % (10-s average, +// 2.5–5.7 s of full stall); the same copy inside a container capped at 512 MB: 0.9–1.0 %, 0.3–0.4 s. So the copy runs +// in a capped container of the controller's own image (it carries rsync) — ONLY when both paths are under /mnt/, +// where the controller and the Docker daemon see the same paths. Any other path (the controller's own data volume) +// differs for the daemon: a capped `--delete` there would mirror an EMPTY folder over the copy. Those paths, a box +// whose image cannot be read, and a container that cannot START (docker's own exit 125) take the plain rsync, as +// before. Pinned by TestR1214_*. func rsyncMirror(src, dst string) error { if err := os.MkdirAll(dst, 0755); err != nil { return fmt.Errorf("mkdir %s: %w", dst, err) @@ -777,7 +789,23 @@ func rsyncMirror(src, dst string) error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() // Trailing slashes: copy the CONTENTS of src into dst. - cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", strings.TrimRight(src, "/")+"/", strings.TrimRight(dst, "/")+"/") + s, d := strings.TrimRight(src, "/"), strings.TrimRight(dst, "/") + rs := []string{"-a", "--delete", s + "/", d + "/"} + if tier2CappedPath(s) && tier2CappedPath(d) { + if img := tier2SelfImage(); img != "" { + args := append([]string{"run", "--rm", "--memory", tier2CopyMemCap, "--memory-swap", tier2CopyMemCap, + "--network", "none", "-v", s + ":" + s + ":ro", "-v", d + ":" + d, "--entrypoint", "rsync", img}, rs...) + out, code, err := tier2RunCapped(ctx, args) + if err == nil { + return nil + } + if code != 125 { + return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) + } + tier2Logf("[WARN] [tier2] the capped copy container did not start (%s) — copying in-process, as before (R-1214)", strings.TrimSpace(string(out))) + } + } + cmd := exec.CommandContext(ctx, "rsync", rs...) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) @@ -785,6 +813,39 @@ func rsyncMirror(src, dst string) error { return nil } +// tier2CopyMemCap bounds the capped copy's memory: rsync itself needs a few MB per 100 000 files; the rest is the page +// cache the copy creates, reclaimed inside the cap instead of from the apps. +const tier2CopyMemCap = "512m" + +// tier2CappedPath: a path the controller and the Docker daemon see identically (the guest's /mnt is bind-mounted into +// the controller at /mnt). Never a relative path, never `..`. +func tier2CappedPath(p string) bool { + return strings.HasPrefix(p, tier2CappedRoot) && filepath.Clean(p) == p +} + +// tier2CappedRoot is "/mnt/" in production (a test seam: tests point it at a temp dir). +var tier2CappedRoot = "/mnt/" + +// Seams (tests never reach Docker — tests that do act on the host they run on). +var ( + tier2SelfImage = func() string { + out, err := dockerexec.Command("docker", "inspect", "felhom-controller", "--format", "{{.Config.Image}}").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) + } + tier2RunCapped = func(ctx context.Context, args []string) ([]byte, int, error) { + out, err := dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput() + var ee *exec.ExitError + if errors.As(err, &ee) { + return out, ee.ExitCode(), err + } + return out, 0, err + } + tier2Logf = log.Printf +) + // tier2UnitPreservedWarning is the customer-facing half of the R-403 skip. It rides in // CrossDriveBackup.LastWarning, which the per-app card already renders, so the refusal reaches the // SURFACE and not only the log — the shape recordTier2NoTarget established. diff --git a/controller/scripts/docker_run_volume_path_gate.py b/controller/scripts/docker_run_volume_path_gate.py index 3449229..3ffad6d 100644 --- a/controller/scripts/docker_run_volume_path_gate.py +++ b/controller/scripts/docker_run_volume_path_gate.py @@ -14,6 +14,12 @@ ROOTS = ["internal", "cmd"] # (file suffix, substring that must appear on the "-v" line, why it is safe) ALLOWLIST = [ + ("internal/backup/tier2.go", '"-v", s + ":" + s + ":ro", "-v", d + ":" + d', + "R-1214 (2026-10-22): the second-drive copy's memory-capped rsync container. Reached ONLY when both paths pass " + "tier2CappedPath — under /mnt/, which the golden deployment bind-mounts into the controller at the IDENTICAL " + "path (container-inspect on 9202 2026-10-22: /mnt -> /mnt) — so the daemon resolves them correctly; every other " + "path (the controller's own data volume) takes the in-process rsync. Pinned by " + "TestR1214_APathOutsideMntNeverGoesToDocker"), ("internal/appbackup/dbdump.go", '"psql", "-v"', "psql's own -v flag (ON_ERROR_STOP) — not a docker mount at all"), ("internal/appexport/estimate.go", '"-v", volumeName+":/vol:ro"',