diff --git a/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth.py b/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth.py new file mode 100644 index 0000000..1b3cdfa --- /dev/null +++ b/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Campaign 10 — monotonic-growth sampler (Campaign 2's controller_rss.tsv precedent, widened). + +WHY THIS EXISTS, separately from the invariants: I1-I11 are CORRECTNESS checks — they answer +"is the system telling the truth this cycle". Every one of them can pass 45 times in a row while +disk usage, snapshot count, log volume or RSS climbs steadily. Accumulation is the half a +targeted session structurally cannot test, and it is the whole point of depth. + +Runs as its own process alongside c10run.py so the in-flight soak does not have to be restarted. +Reads the runner's status.txt so every sample is attributed to a cycle. Tolerates the box being +down (hard-reset / reboot atoms) by recording NA rather than dying. + +One row per sample, long format: epoch, iso, cycle, metric, value. +""" +import os, re, subprocess, time + +EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31" +STATE = os.path.join(EV, "state") +OUT = os.path.join(STATE, "growth.tsv") +STATUS = os.path.join(STATE, "status.txt") +INTERVAL = 90 + +HOME = os.path.expanduser("~/.config/campaign10") +import json +HOSTPW = json.load(open(os.path.join(HOME, "host-recovery.json")))["password"] + + +def box(cmd, timeout=60): + try: + p = subprocess.run(["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "ConnectTimeout=12", "-J", "demo-hp", "root@192.168.0.105", + "LC_ALL=C " + cmd], + capture_output=True, text=True, timeout=timeout, + env=dict(os.environ, SSHPASS=HOSTPW, LC_ALL="C")) + return p.stdout.strip() + except Exception: + return "" + + +def hp(cmd, timeout=60): + try: + p = subprocess.run(["ssh", "-o", "ConnectTimeout=12", "demo-hp", "LC_ALL=C " + cmd], + capture_output=True, text=True, timeout=timeout) + return p.stdout.strip() + except Exception: + return "" + + +def cycle_now(): + try: + m = re.search(r"cycle=(\d+)", open(STATUS).read()) + return m.group(1) if m else "?" + except Exception: + return "?" + + +def num(s, default="NA"): + m = re.search(r"-?\d+(?:\.\d+)?", s or "") + return m.group(0) if m else default + + +def sample(): + """One batched round-trip per host keeps the load off the box the soak is abusing.""" + rows = {} + + # --- inside the guest: controller/app footprint, log volume, docker object counts --- + g = box( + "pct exec 9201 -- bash -c '" + "echo RSS_CTRL=$(docker stats --no-stream --format \"{{.Name}} {{.MemUsage}}\" felhom-controller 2>/dev/null | awk \"{print \\$2}\"); " + "echo FD_CTRL=$(docker inspect -f {{.State.Pid}} felhom-controller 2>/dev/null | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); " + "echo CTRL_RESTARTS=$(docker inspect -f {{.RestartCount}} felhom-controller 2>/dev/null); " + "echo N_CONTAINERS=$(docker ps -q 2>/dev/null | wc -l); " + "echo N_VOLUMES=$(docker volume ls -q 2>/dev/null | wc -l); " + "echo N_IMAGES=$(docker images -q 2>/dev/null | wc -l); " + "echo LOGS_MB=$(du -sm /var/lib/docker/containers 2>/dev/null | cut -f1); " + "echo JOURNAL_MB=$(journalctl --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); " + "echo BACKUPS_MB=$(du -sm /mnt/sys_drive/felhom-data/backups 2>/dev/null | cut -f1); " + "echo PGVOL_MB=$(du -sm /var/lib/docker/volumes/rallly_rallly_postgres_data 2>/dev/null | cut -f1); " + "'") + for line in g.splitlines(): + if "=" in line: + k, _, v = line.partition("=") + rows[k.strip()] = num(v) if k.strip() != "JOURNAL_MB" else (v.strip() or "NA") + + # --- on the PVE host (the VM): agent footprint + the drives themselves --- + h = box( + "bash -c '" + "echo RSS_AGENT_KB=$(ps -o rss= -C felhom-agent 2>/dev/null | head -1); " + "echo FD_AGENT=$(pgrep -x felhom-agent | head -1 | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); " + "echo ROOT_USED_MB=$(df -BM --output=used / | tail -1 | tr -dc 0-9); " + # MUST gate on mountpoint: when a drive is detached, /mnt/ reverts to a plain + # directory on root and df silently reports the ROOT filesystem — which read as a + # 623->5667 MB "jump" (exactly ROOT_USED_MB) in the first samples. Same class of error + # as the agent's exactMount check, in the measurement code. + "echo MENTES_USED_MB=$(mountpoint -q /mnt/mentes && df -BM --output=used /mnt/mentes | tail -1 | tr -dc 0-9 || echo NA); " + "echo ADATOK_USED_MB=$(mountpoint -q /mnt/adatok && df -BM --output=used /mnt/adatok | tail -1 | tr -dc 0-9 || echo NA); " + "echo VZDUMP_N=$(mountpoint -q /mnt/mentes && ls -1 /mnt/mentes/dump 2>/dev/null | wc -l || echo NA); " + "echo AGENT_JOURNAL_MB=$(journalctl -u felhom-agent --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); " + "'") + for line in h.splitlines(): + if "=" in line: + k, _, v = line.partition("=") + rows[k.strip()] = num(v) if k.strip() != "AGENT_JOURNAL_MB" else (v.strip() or "NA") + + # --- on demo-hp: the qcow2 files actually growing on the host --- + d = hp("du -sm /mnt/nvme-1tb/images/311 2>/dev/null | cut -f1; df -BM --output=avail /mnt/nvme-1tb | tail -1 | tr -dc 0-9") + parts = [p for p in d.splitlines() if p.strip()] + rows["VMDISK_MB"] = num(parts[0]) if len(parts) > 0 else "NA" + rows["NVME_AVAIL_MB"] = num(parts[1]) if len(parts) > 1 else "NA" + return rows + + +def main(): + new = not os.path.exists(OUT) + with open(OUT, "a") as fh: + if new: + fh.write("epoch\tiso\tcycle\tmetric\tvalue\n") + while True: + try: + r = sample() + ep = int(time.time()) + iso = time.strftime("%FT%TZ", time.gmtime()) + cy = cycle_now() + with open(OUT, "a") as fh: + for k in sorted(r): + fh.write("%d\t%s\t%s\t%s\t%s\n" % (ep, iso, cy, k, r[k])) + except Exception as e: + with open(OUT, "a") as fh: + fh.write("%d\t%s\t%s\tSAMPLER_ERROR\t%s\n" + % (int(time.time()), time.strftime("%FT%TZ", time.gmtime()), cycle_now(), e)) + time.sleep(INTERVAL) + + +if __name__ == "__main__": + main() diff --git a/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth_report.py b/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth_report.py new file mode 100644 index 0000000..3e42c37 --- /dev/null +++ b/documentation/tests/campaign10-evidence-2026-07-31/runner/c10growth_report.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Campaign 10 — turn growth.tsv into a verdict per metric (Campaign 2's table shape). + +The question is NOT "did it grow" — a backup soak SHOULD grow backup storage. It is +"does anything grow WITHOUT BOUND, per cycle, in a way 45 cycles would not survive". +So each metric gets: start, end, min, max, per-cycle slope (least squares over cycle +number), and a verdict that distinguishes expected accumulation from a leak. +""" +import collections, os, sys + +STATE = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31/state" +SRC = os.path.join(STATE, "growth.tsv") + +# metrics where monotonic growth is a LEAK, vs where it is expected accumulation +LEAK_IF_GROWING = { + "RSS_CTRL": "controller memory", "RSS_AGENT_KB": "agent memory", + "FD_CTRL": "controller file descriptors", "FD_AGENT": "agent file descriptors", + "N_VOLUMES": "docker volumes (redeploys must not orphan)", + "N_IMAGES": "docker images", "N_CONTAINERS": "running containers", + "CTRL_RESTARTS": "controller restart count", +} +EXPECTED_GROWTH = { + "BACKUPS_MB": "recovery units", "MENTES_USED_MB": "backup target", + "VZDUMP_N": "whole-guest dumps", "VMDISK_MB": "qcow2 on the host", + "LOGS_MB": "docker logs", "JOURNAL_MB": "guest journal", + "AGENT_JOURNAL_MB": "agent journal", "ROOT_USED_MB": "guest root fs", + "ADATOK_USED_MB": "data drive", "PGVOL_MB": "postgres volume", + "NVME_AVAIL_MB": "host free space (falls)", +} + + +def tofloat(v): + v = (v or "").strip().rstrip("MG") + try: + return float(v) + except Exception: + return None + + +def main(): + series = collections.defaultdict(list) # metric -> [(cycle, value)] + if not os.path.exists(SRC): + print("no growth.tsv yet"); return + for line in open(SRC): + f = line.rstrip("\n").split("\t") + if len(f) != 5 or f[0] == "epoch": + continue + _, _, cy, metric, val = f + v = tofloat(val) + try: + c = int(cy) + except Exception: + continue + if v is not None: + series[metric].append((c, v)) + + print("%-18s %10s %10s %10s %10s %12s %s" % + ("metric", "start", "end", "min", "max", "slope/cycle", "verdict")) + print("-" * 104) + concerns = [] + for m in sorted(series): + pts = series[m] + if len(pts) < 3: + continue + xs = [p[0] for p in pts]; ys = [p[1] for p in pts] + n = len(xs); mx = sum(xs) / n; my = sum(ys) / n + den = sum((x - mx) ** 2 for x in xs) + slope = (sum((xs[i] - mx) * (ys[i] - my) for i in range(n)) / den) if den else 0.0 + start, end, lo, hi = ys[0], ys[-1], min(ys), max(ys) + span = max(xs) - min(xs) + # projection to cycle 45 from the last observed point + proj = end + slope * max(0, 45 - max(xs)) + if m in LEAK_IF_GROWING: + rel = (slope * max(span, 1)) / (abs(start) or 1) + verdict = "LEAK-SUSPECT" if (slope > 0 and rel > 0.25) else "stable — no leak" + if verdict.startswith("LEAK"): + concerns.append((m, slope, end, proj)) + else: + label = EXPECTED_GROWTH.get(m, "") + verdict = "expected accumulation (%s); proj@c45=%.0f" % (label, proj) + print("%-18s %10.1f %10.1f %10.1f %10.1f %12.2f %s" % (m, start, end, lo, hi, slope, verdict)) + print() + if concerns: + print("CONCERNS (growth where growth is a leak):") + for m, s, e, p in concerns: + print(" %s slope=%.2f/cycle now=%.1f projected@c45=%.1f" % (m, s, e, p)) + else: + print("No leak-class metric grew materially.") + + +if __name__ == "__main__": + main()