diff --git a/CHANGELOG.md b/CHANGELOG.md index b698f31..a172ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2026-10-06 (night, held) — the memory watch records swap (R-733, first part) + +**What runs on a box changed:** nothing. Test tools only. (Held with the night's other catalog commits.) + +- `scripts/upgrade-test.py`: each container's snapshot reads `memory.swap.current` and `memory.swap.peak`; the watch's per-container record carries `swap_peak` (the largest sampled reading or the kernel's own peak; None when nothing was readable, never 0), and the memory record carries `venue_swap_bytes` (SwapTotal of the venue). Read tonight: the bench has 0 swap, scratch 9202 has 512 MiB. Additive — no verdict or mark reads them. +- Tests: SwapRecorded (4), red-proved. +- **Left on the row:** the golden's guest `swap` read into its bake evidence (agent repo), the box walk's record, and the decision whether proofs run with swap off. + ## 2026-10-06 (night, held) — gramps-web runs 2 web workers, not 8 (R-806) **What runs on a box changed:** gramps-web's environment. **Held on branch `night-held-2026-10-06`**, not `main` tonight. diff --git a/scripts/test_upgrade_bench.py b/scripts/test_upgrade_bench.py index 989a366..22a9639 100644 --- a/scripts/test_upgrade_bench.py +++ b/scripts/test_upgrade_bench.py @@ -495,5 +495,27 @@ class SecretHygiene(unittest.TestCase): self.assertEqual(ut.redact("x abcdefgh123 y", ["abcdefgh123", "abcdefgh"]), "x y") +class SwapRecorded(unittest.TestCase): + """R-733: the memory record carries the venue's swap and each container's swap peak. + + COMPANION RED-PROOF (observed 2026-10-06): make venue_swap_bytes return None always → test_venue_swap_read + fails; make swap_peak_of ignore the kernel's swap_peak → test_kernel_peak_counts fails. Restored.""" + + def test_venue_swap_read(self): + self.assertEqual(ut.venue_swap_bytes("MemTotal: 1 kB\nSwapTotal: 524288 kB\nSwapFree: 1 kB\n"), 512 * 2**20) + self.assertEqual(ut.venue_swap_bytes("SwapTotal: 0 kB\n"), 0) + + def test_venue_swap_unreadable_is_none_not_zero(self): + self.assertIsNone(ut.venue_swap_bytes("MemTotal: 1 kB\n")) + + def test_kernel_peak_counts(self): + samples = [{"containers": {"app": {"swap": 10}}}, {"containers": {"app": {"swap": 30}}}] + self.assertEqual(ut.swap_peak_of(samples, "app", {"swap": 5, "swap_peak": 99}), 99) + self.assertEqual(ut.swap_peak_of(samples, "app", {"swap": 5, "swap_peak": None}), 30) + + def test_nothing_readable_is_none(self): + self.assertIsNone(ut.swap_peak_of([{"containers": {}}], "app", {"swap": None, "swap_peak": None})) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/upgrade-test.py b/scripts/upgrade-test.py index 494c327..f5f34d4 100755 --- a/scripts/upgrade-test.py +++ b/scripts/upgrade-test.py @@ -441,6 +441,28 @@ def _stat_anon(cg: str): return None +def venue_swap_bytes(meminfo: str | None = None): + """R-733: the venue's own swap (SwapTotal of /proc/meminfo, in bytes) — None when it cannot be read. + The bench guest had NONE while every box guest reads `swap: 512`: immich passed on 9202 by swapping + ~108 MB and was OOM-killed 61-104 times on the bench. Every memory record now carries the venue's + swap, so a proof that leaned on swap says so.""" + try: + text = meminfo if meminfo is not None else open("/proc/meminfo").read() + except OSError: + return None + m = re.search(r"^SwapTotal:\s+(\d+)\s*kB", text, re.M) + return int(m.group(1)) * 1024 if m else None + + +def swap_peak_of(samples, name: str, end: dict): + """R-733: the most swap a container was seen using — the largest of the sampled `swap` readings, the + end reading and the kernel's own `swap_peak` (memory.swap.peak, kernel 6.5+). None when nothing was + readable: "could not look" never reads as 0.""" + vals = [smp["containers"].get(name, {}).get("swap") for smp in samples] + [end.get("swap"), end.get("swap_peak")] + vals = [v for v in vals if isinstance(v, int)] + return max(vals) if vals else None + + def memory_snapshot(project: str, workdir: Path): """One reading per container: usage, peak, limit, kernel OOM kills, restarts, OOMKilled.""" out = {} @@ -461,6 +483,9 @@ def memory_snapshot(project: str, workdir: Path): # too, which the kernel reclaims before it kills anything (night 2026-09-23, nextcloud # read 100 % with 0 kills). Reported, not yet used for the verdict or the mark. "anon": _stat_anon(cg) if cg else None, + # R-733: swap beside anon — a box guest has 512 MiB swap, the bench had none. + "swap": _read_int(os.path.join(cg, "memory.swap.current")) if cg else None, + "swap_peak": _read_int(os.path.join(cg, "memory.swap.peak")) if cg else None, "oom_kill": _oom_kills(cg) if cg else None, "restarts": (info.get("State") or {}).get("RestartCount", 0), "oomkilled_flag": (info.get("State") or {}).get("OOMKilled"), @@ -568,6 +593,7 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P "peak_pct": round(pk / lim, 3) if (pk and lim) else None, "anon_peak_sampled": anon_max or None, "anon_peak_pct": round(anon_max / lim, 3) if (anon_max and lim) else None, + "swap_peak": swap_peak_of(samples, n, v), "oom_kills": None if v["oom_kill"] is None else v["oom_kill"] - (b.get("oom_kill") or 0), "restarts": (v["restarts"] or 0) - (b.get("restarts") or 0), "oomkilled_flag": v["oomkilled_flag"], "measured": v["cgroup"]} @@ -583,6 +609,7 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P rec = {"soak_s": round(time.time() - t0, 1), "requested_s": seconds, "requests": hits["n"], "codes": hits["codes"], "first_kill": first_bad, "containers": per, "unmeasured": [n for n, p in per.items() if not p["measured"]], + "venue_swap_bytes": venue_swap_bytes(), "load": load_verdict(hits["n"], hits["codes"])} marks = ["memory_tight"] if tight and not killed else [] say(f"memory watch: killed={killed} tight={tight} requests={hits['n']} codes={hits['codes']}")