felhom.eu/documentation/audits/pg-last-six-2026-09-30/tools/partC.py

#!/usr/bin/env python3
"""partC.py setup|run|restore — R-687 item 4 by day on demo-hp guest 9201: the full-system backup WAITS for the
automatic update leg (`09` decision 20), seen live.

The gate's wait is on the SCHEDULED path only (quiesce.go runOnce), so it needs, at one poll: the whole-guest tier
DUE (local tier: last success 2026-09-29 04:42, cadence 24 h → due), "now" inside [W+2h, W+6h) and the leg active
(stacks.UpdateLegState — a MANUAL chain's leg sets the same flag, unattended.go runUpdateLeg/setLegFlags).
The ORDER matters: the tier is already due, so the window may move ONLY once the leg is running — otherwise the next
poll starts the backup during the chain's own dump/off-site legs. So:
  setup   — controller.yaml saved as controller.yaml.pre-gate-day; `quiesce: poll_interval: 1m` added; restart.
  run     — press the night chain (POST /api/debug/backup/night-chain); the moment `[update-leg] started (manual-chain)`
            is in the log, POST /backups/window so that now = W+2h05m (the product's own form); then watch for the
            deferral line, the leg's end line, the whole-guest backup starting AFTER it, and its result.
  restore — the saved controller.yaml back; `backup_window_start` removed from settings.json with the controller
            STOPPED (it was never set: the box ran on controller.yaml's db_dump_schedule 02:30); start; read back.
Evidence: C/ (every step appends). Never prints a secret.
"""
import json, os, re, subprocess, sys, time
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
os.environ["GUEST"] = "9201"
import walk as w

EVD = f"{w.EV}/C"; os.makedirs(EVD, exist_ok=True)
VOL = "/var/lib/docker/volumes/felhom-controller-data/_data"
BUD = ZoneInfo("Europe/Budapest")
log = open(f"{EVD}/C-{sys.argv[1]}.txt", "a", buffering=1)


def say(*a):
    w.say(*a); log.write(f"{datetime.now().strftime('%H:%M:%S')} " + " ".join(map(str, a)) + "\n")


def logs_since(since, pat):
    return w.guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -E '{pat}' | grep -v DEBUG | cut -c1-330")


def form_post(path, body):
    sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
    csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
    r = w.sh(["curl", "-sk", "-o", "/dev/null", "-D", "-", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}",
              "-H", "Content-Type: application/x-www-form-urlencoded", "--data", body, f"{w.BASE}{path}"])
    st = re.search(r"^HTTP/\S+ (\d+)", r.stdout or "", re.M)
    loc = re.search(r"(?im)^location:\s*(\S+)", r.stdout or "")
    return (st.group(1) if st else "?"), (loc.group(1) if loc else "")


act = sys.argv[1]
if act == "setup":
    say("# setup", datetime.now(BUD).isoformat())
    say(w.guest(f"""set -e
test -f {VOL}/controller.yaml.pre-gate-day || cp -p {VOL}/controller.yaml {VOL}/controller.yaml.pre-gate-day
grep -q '^quiesce:' {VOL}/controller.yaml && echo 'quiesce section EXISTS — refusing to edit' && exit 1
printf 'quiesce:\\n    poll_interval: 1m\\n' >> {VOL}/controller.yaml
diff {VOL}/controller.yaml.pre-gate-day {VOL}/controller.yaml || true
cp -p {VOL}/data/settings.json {VOL}/data/settings.json.pre-gate-day
docker restart felhom-controller >/dev/null; sleep 20
docker logs --since 1m felhom-controller 2>&1 | grep -E 'quiesce\\] loop started|update-leg|backup window' | cut -c1-200"""))
elif act == "run":
    w.login()
    t0 = w.guest("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
    say("# run", datetime.now(BUD).isoformat(), "since", t0)
    c, d = w.ctl("POST", "/api/debug/backup/night-chain")
    say("night-chain press ->", c, json.dumps(d)[:300])
    if c not in ("200", "202"):
        sys.exit("the chain was refused — waited out, never worked around; rerun later")
    moved = False
    deadline = time.time() + 3600
    seen = set()
    while time.time() < deadline:
        out = logs_since(t0, r"night-chain|update-leg|\[quiesce\]|update (bookstack|kimai)|whole-guest|backup window")
        for line in out.splitlines():
            if line not in seen:
                seen.add(line); log.write("  LOG " + line + "\n")
        if not moved and "[update-leg] started" in out:
            now = datetime.now(BUD)
            W = (now - timedelta(hours=2, minutes=5)).strftime("%H:%M")
            code, loc = form_post("/backups/window", f"window_start={W}")
            moved = True
            say(f"leg started — window moved to {W} (now {now.strftime('%H:%M:%S')} = W+2h05m): POST /backups/window -> {code} {loc}")
        # quiesce.go:554/562/628/631/536 — the job's own start/done/failed lines, then the unquiesce (quiesce.go:492)
        if re.search(r"\[quiesce\] tier \S+: backup job \S+ (done|failed)|\[quiesce\] start backup on tier", out):
            time.sleep(60)
            break
        time.sleep(3)
    out = logs_since(t0, r"night-chain|update-leg|\[quiesce\]|whole-guest")
    open(f"{EVD}/C-run-log.txt", "w").write(out)
    say("lines kept:", len(out.splitlines()), "→ C/C-run-log.txt")
elif act == "restore":
    say("# restore", datetime.now(BUD).isoformat())
    say(w.guest(f"""set -e
cp -p {VOL}/controller.yaml.pre-gate-day {VOL}/controller.yaml
docker stop felhom-controller >/dev/null
python3 - <<'PY'
import json
p = "{VOL}/data/settings.json"
d = json.load(open(p))
print("settings backup_window_start before restore:", d.get("backup_window_start"))
d.pop("backup_window_start", None)
json.dump(d, open(p, "w"), indent=2)
print("settings backup_window_start after restore:", d.get("backup_window_start"))
PY
docker start felhom-controller >/dev/null; sleep 20
echo "controller.yaml vs the saved copy:"; diff {VOL}/controller.yaml {VOL}/controller.yaml.pre-gate-day && echo IDENTICAL
grep -c '^quiesce:' {VOL}/controller.yaml || true
grep -n 'db_dump_schedule' {VOL}/controller.yaml
docker logs --since 1m felhom-controller 2>&1 | grep -E 'quiesce\\] loop started|scheduled|window' | cut -c1-220"""))