#!/usr/bin/env python3
"""Campaign 10 — monotonic-growth sampler (Campaign 2's controller_rss.tsv precedent, widened).

WHY THIS EXISTS, separately from the invariants: I1-I11 are CORRECTNESS checks — they answer
"is the system telling the truth this cycle". Every one of them can pass 45 times in a row while
disk usage, snapshot count, log volume or RSS climbs steadily. Accumulation is the half a
targeted session structurally cannot test, and it is the whole point of depth.

Runs as its own process alongside c10run.py so the in-flight soak does not have to be restarted.
Reads the runner's status.txt so every sample is attributed to a cycle. Tolerates the box being
down (hard-reset / reboot atoms) by recording NA rather than dying.

One row per sample, long format: epoch, iso, cycle, metric, value.
"""
import os, re, subprocess, time

EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31"
STATE = os.path.join(EV, "state")
OUT = os.path.join(STATE, "growth.tsv")
STATUS = os.path.join(STATE, "status.txt")
INTERVAL = 90

HOME = os.path.expanduser("~/.config/campaign10")
import json
HOSTPW = json.load(open(os.path.join(HOME, "host-recovery.json")))["password"]


def box(cmd, timeout=60):
    try:
        p = subprocess.run(["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no",
                            "-o", "ConnectTimeout=12", "-J", "demo-hp", "root@192.168.0.105",
                            "LC_ALL=C " + cmd],
                           capture_output=True, text=True, timeout=timeout,
                           env=dict(os.environ, SSHPASS=HOSTPW, LC_ALL="C"))
        return p.stdout.strip()
    except Exception:
        return ""


def hp(cmd, timeout=60):
    try:
        p = subprocess.run(["ssh", "-o", "ConnectTimeout=12", "demo-hp", "LC_ALL=C " + cmd],
                           capture_output=True, text=True, timeout=timeout)
        return p.stdout.strip()
    except Exception:
        return ""


def cycle_now():
    try:
        m = re.search(r"cycle=(\d+)", open(STATUS).read())
        return m.group(1) if m else "?"
    except Exception:
        return "?"


def num(s, default="NA"):
    m = re.search(r"-?\d+(?:\.\d+)?", s or "")
    return m.group(0) if m else default


def sample():
    """One batched round-trip per host keeps the load off the box the soak is abusing."""
    rows = {}

    # --- inside the guest: controller/app footprint, log volume, docker object counts ---
    g = box(
        "pct exec 9201 -- bash -c '"
        "echo RSS_CTRL=$(docker stats --no-stream --format \"{{.Name}} {{.MemUsage}}\" felhom-controller 2>/dev/null | awk \"{print \\$2}\"); "
        "echo FD_CTRL=$(docker inspect -f {{.State.Pid}} felhom-controller 2>/dev/null | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); "
        "echo CTRL_RESTARTS=$(docker inspect -f {{.RestartCount}} felhom-controller 2>/dev/null); "
        "echo N_CONTAINERS=$(docker ps -q 2>/dev/null | wc -l); "
        "echo N_VOLUMES=$(docker volume ls -q 2>/dev/null | wc -l); "
        "echo N_IMAGES=$(docker images -q 2>/dev/null | wc -l); "
        "echo LOGS_MB=$(du -sm /var/lib/docker/containers 2>/dev/null | cut -f1); "
        "echo JOURNAL_MB=$(journalctl --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); "
        "echo BACKUPS_MB=$(du -sm /mnt/sys_drive/felhom-data/backups 2>/dev/null | cut -f1); "
        "echo PGVOL_MB=$(du -sm /var/lib/docker/volumes/rallly_rallly_postgres_data 2>/dev/null | cut -f1); "
        "'")
    for line in g.splitlines():
        if "=" in line:
            k, _, v = line.partition("=")
            rows[k.strip()] = num(v) if k.strip() != "JOURNAL_MB" else (v.strip() or "NA")

    # --- on the PVE host (the VM): agent footprint + the drives themselves ---
    h = box(
        "bash -c '"
        "echo RSS_AGENT_KB=$(ps -o rss= -C felhom-agent 2>/dev/null | head -1); "
        "echo FD_AGENT=$(pgrep -x felhom-agent | head -1 | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); "
        "echo ROOT_USED_MB=$(df -BM --output=used / | tail -1 | tr -dc 0-9); "
        # MUST gate on mountpoint: when a drive is detached, /mnt/<name> reverts to a plain
        # directory on root and df silently reports the ROOT filesystem — which read as a
        # 623->5667 MB "jump" (exactly ROOT_USED_MB) in the first samples. Same class of error
        # as the agent's exactMount check, in the measurement code.
        "echo MENTES_USED_MB=$(mountpoint -q /mnt/mentes && df -BM --output=used /mnt/mentes | tail -1 | tr -dc 0-9 || echo NA); "
        "echo ADATOK_USED_MB=$(mountpoint -q /mnt/adatok && df -BM --output=used /mnt/adatok | tail -1 | tr -dc 0-9 || echo NA); "
        "echo VZDUMP_N=$(mountpoint -q /mnt/mentes && ls -1 /mnt/mentes/dump 2>/dev/null | wc -l || echo NA); "
        "echo AGENT_JOURNAL_MB=$(journalctl -u felhom-agent --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); "
        "'")
    for line in h.splitlines():
        if "=" in line:
            k, _, v = line.partition("=")
            rows[k.strip()] = num(v) if k.strip() != "AGENT_JOURNAL_MB" else (v.strip() or "NA")

    # --- on demo-hp: the qcow2 files actually growing on the host ---
    d = hp("du -sm /mnt/nvme-1tb/images/311 2>/dev/null | cut -f1; df -BM --output=avail /mnt/nvme-1tb | tail -1 | tr -dc 0-9")
    parts = [p for p in d.splitlines() if p.strip()]
    rows["VMDISK_MB"] = num(parts[0]) if len(parts) > 0 else "NA"
    rows["NVME_AVAIL_MB"] = num(parts[1]) if len(parts) > 1 else "NA"
    return rows


def main():
    new = not os.path.exists(OUT)
    with open(OUT, "a") as fh:
        if new:
            fh.write("epoch\tiso\tcycle\tmetric\tvalue\n")
    while True:
        try:
            r = sample()
            ep = int(time.time())
            iso = time.strftime("%FT%TZ", time.gmtime())
            cy = cycle_now()
            with open(OUT, "a") as fh:
                for k in sorted(r):
                    fh.write("%d\t%s\t%s\t%s\t%s\n" % (ep, iso, cy, k, r[k]))
        except Exception as e:
            with open(OUT, "a") as fh:
                fh.write("%d\t%s\t%s\tSAMPLER_ERROR\t%s\n"
                         % (int(time.time()), time.strftime("%FT%TZ", time.gmtime()), cycle_now(), e))
        time.sleep(INTERVAL)


if __name__ == "__main__":
    main()