felhom.eu/documentation/audits/update-night-2026-09-21/phase5_teardown.py

#!/usr/bin/env python3
"""Phase 5 — teardown, three layers, stated. Plus Gitea.

R-320: evidence is copied off at the end of the phase that produced it, before any revert. By the
time this runs every phase's evidence is already in this directory; this file only tears down, and
it PROVES each layer rather than asserting it.

    MACHINE (9202) every throwaway app removed THROUGH THE PRODUCT with its data; the image store
                  container and volume gone; drill images removed BY NAME (never `prune`);
                  controller.yaml restored from the saved copy; the controller restarted; and
                  `git.repo_url` READ BACK AND QUOTED as the LIVE catalog, with one sync + rescan
                  showing only the protected infra containers and no badge.
    HOST (demo-hp) `pct list` and `pvesm status` before and after; guest 9201 untouched.
    HUB           nothing provisioned; the floor's state stated.
    GITEA         the drill repo KEPT, private, RESET to the live catalog's main; and the live
                  catalog's own main hash plus a diff of every `image:` line — expected: identical.
"""
import json, os, subprocess, sys, time

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import walk as w   # noqa: E402

OUT = os.path.join(HERE, "teardown")
KEEP = {"traefik", "filebrowser", "felhom-controller"}      # the protected infra containers
LIVE_REPO = "https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git"
LIVE_DIR = "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu"
DRILL = w.DRILL
SC = w.SC


def sec(name, text):
    open(f"{OUT}/{name}", "w").write(text)
    w.say(f"  -> {name}")
    return text


def main():
    os.makedirs(OUT, exist_ok=True)
    w.login()
    w.say("==== Phase 5: teardown, three layers")
    rec = {}

    # ---------------------------------------------------------------- BEFORE, host layer
    host_before = w.sh(["ssh", "-o", "ConnectTimeout=20", w.HP,
                        "pct list; echo '--- pvesm'; pvesm status"], timeout=180).stdout or ""
    sec("00-host-before.txt", host_before)
    rec["host_before"] = host_before

    # ---------------------------------------------------------------- MACHINE: the apps
    _, d = w.ctl("GET", "/api/stacks")
    deployed = [s["name"] for s in (d.get("data") or []) if s.get("deployed")]
    w.say(f"  [M] throwaway apps still deployed: {deployed}")
    removed = {}
    for n in deployed:
        if n in KEEP:
            continue
        removed[n] = w.remove(n)
    rec["apps_removed"] = removed

    # ---------------------------------------------------------------- MACHINE: the image store
    store = w.guest("""
echo "=== registry container + volume, removed BY NAME (never a prune)"
docker rm -f drill-registry 2>&1 | head -2
docker volume rm drill-registry-data 2>&1 | head -2
echo "=== drill images, removed BY NAME"
for i in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^localhost:5000/drill/'); do
  echo -n "$i -> "; docker rmi "$i" >/dev/null 2>&1 && echo removed || echo "in use / already gone"
done
docker rmi registry:2 >/dev/null 2>&1 && echo "registry:2 removed" || echo "registry:2 kept/in use"
echo "=== anything left that says drill?"
docker images --format '{{.Repository}}:{{.Tag}}' | grep -i drill || echo "(none)"
docker ps -a --format '{{.Names}}' | grep -i drill || echo "(no drill containers)"
echo "=== NOTHING WAS PRUNED — this is the full image list, for the record"
docker images --format '{{.Repository}}:{{.Tag}} {{.Size}}' | sort | head -40
""", timeout=900)
    sec("01-image-store-removed.txt", store)

    # ---------------------------------------------------------------- MACHINE: the config back
    V = "/var/lib/docker/volumes/felhom-controller-data/_data"
    back = w.guest(f"""
echo "=== restoring controller.yaml from the pre-update-night copy"
ls -la {V}/controller.yaml {V}/controller.yaml.pre-update-night
cp {V}/controller.yaml.pre-update-night {V}/controller.yaml && echo "restored"
echo "=== the git section, read back (token redacted by this script, not by the box)"
sed -n '/^git:/,/^hub:/p' {V}/controller.yaml | sed 's/token: ".*"/token: "<redacted>"/'
echo "=== removing the drill catalog cache so the next sync clones the LIVE repo (R-615)"
rm -rf {V}/data/catalog-cache
systemctl restart felhom-controller-bootstrap.service
sleep 25
docker ps --filter name=felhom-controller --format '{{{{.Image}}}} {{{{.Status}}}}'
echo "=== the cache's origin, READ BACK — this is the quote the brief asks for"
git -C {V}/data/catalog-cache remote -v 2>/dev/null | sed 's#://[^@]*@#://#'
git -C {V}/data/catalog-cache log --oneline -1 2>/dev/null
""", timeout=900)
    sec("02-config-restored.txt", back)
    rec["repo_url_read_back"] = LIVE_REPO in back
    w.say(f"  [M] git.repo_url reads back as the LIVE catalog: {rec['repo_url_read_back']}")

    # ---------------------------------------------------------------- MACHINE: sync, rescan, badges
    w.login()
    w.ctl("POST", "/api/sync")
    time.sleep(3)
    w.ctl("POST", "/api/stacks/rescan")
    time.sleep(3)
    _, d = w.ctl("GET", "/api/stacks")
    left = []
    for s in (d.get("data") or []):
        if not s.get("deployed") and s.get("state") == "not_deployed":
            continue
        b = w.badges(s["name"])
        left.append({"name": s["name"], "state": s.get("state"), "deployed": s.get("deployed"),
                     "badge_hu": [x["text"] for x in b["hu"]]})
    rec["still_on_the_box"] = left
    for x in left:
        w.say(f"  [M] {x['name']:20} deployed={x['deployed']} state={x['state']} badge={x['badge_hu']}")
    containers = w.guest("docker ps --format '{{.Names}}\t{{.Image}}\t{{.Status}}'")
    sec("03-containers-after.txt", containers)
    names = {l.split("\t")[0] for l in containers.strip().split("\n") if l.strip()}
    rec["only_protected_infra_left"] = names <= KEEP
    w.say(f"  [M] containers left: {sorted(names)}  only protected infra: {rec['only_protected_infra_left']}")

    # ---------------------------------------------------------------- HOST
    host_after = w.sh(["ssh", "-o", "ConnectTimeout=20", w.HP,
                       "pct list; echo '--- pvesm'; pvesm status; echo '--- 9201 untouched:'; "
                       "pct exec 9201 -- docker ps --format '{{.Names}}' | sort | head -20"],
                      timeout=180).stdout or ""
    sec("04-host-after.txt", host_after)
    rec["host_after"] = host_after
    w.say("  [H] no harness LXC was created tonight, so none was destroyed — "
          "the PostgreSQL rehearsal ran on 9202 itself (stated in the audit)")

    # ---------------------------------------------------------------- HUB
    hp = open(f"{SC}/.hubpw").read().strip()
    conf = w.sh(["curl", "-s", "-u", f":{hp}", "http://10.43.52.34:8080/configuration"],
                timeout=90).stdout or ""
    import re
    floor = re.search(r'min_controller_version" value="([^"]*)"', conf)
    magent = re.search(r'name="min_agent" value="([^"]*)"', conf)
    hosts = w.sh(["curl", "-s", "-u", f":{hp}", "http://10.43.52.34:8080/hosts"],
                 timeout=90).stdout or ""
    nhosts = hosts.count("ONLINE") + hosts.count("DOWN")
    hubtxt = (f"floor = {floor.group(1) if floor else '?'}\n"
              f"min_agent = {magent.group(1) if magent else '?'}\n"
              f"host rows seen = {nhosts}\n"
              "nothing was provisioned at the hub tonight: no customer, no config, no appliance,\n"
              "no binding. The only hub act of the whole night was the floor save in Phase 0.1.\n")
    sec("05-hub.txt", hubtxt)
    rec["hub"] = hubtxt
    w.say("  [U] " + hubtxt.replace("\n", " | "))

    # ---------------------------------------------------------------- GITEA
    w.sh(["git", "-C", LIVE_DIR, "fetch", "-q", "origin"], timeout=180)
    live_main = (w.sh(["git", "-C", LIVE_DIR, "rev-parse", "--short=12", "origin/main"]).stdout or "").strip()
    w.sh(["git", "-C", DRILL, "fetch", "-q", "origin"], timeout=180)
    w.sh(["git", "-C", DRILL, "remote", "remove", "live"], timeout=60)
    w.sh(["git", "-C", DRILL, "remote", "add", "live", LIVE_REPO], timeout=60)
    w.sh(["git", "-C", DRILL, "fetch", "-q", "live", "main"], timeout=300)
    w.sh(["git", "-C", DRILL, "reset", "--hard", "live/main"], timeout=180)
    push = w.sh(["git", "-C", DRILL, "push", "--force", "origin", "main"], timeout=300)
    drill_main = (w.sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout or "").strip()

    # the diff that matters: every image: line, live vs drill
    diff = w.sh(["bash", "-lc",
                 f"diff <(grep -rhoE '^[[:space:]]+image: .*' {LIVE_DIR}/templates/*/docker-compose.yml | sort) "
                 f"<(grep -rhoE '^[[:space:]]+image: .*' {DRILL}/templates/*/docker-compose.yml | sort) "
                 f"&& echo 'IDENTICAL — every image: line matches the live catalog'"],
                timeout=180)
    gitea = (f"live catalog origin/main : {live_main}\n"
             f"drill repo HEAD after reset: {drill_main}\n"
             f"reset+force-push rc={push.returncode}\n\n"
             f"diff of every `image:` line, live vs drill:\n{diff.stdout}{diff.stderr}\n")
    sec("06-gitea.txt", gitea)
    rec["live_main"] = live_main
    rec["drill_main"] = drill_main
    rec["image_lines_identical"] = "IDENTICAL" in diff.stdout
    w.say(f"  [G] live main={live_main}  drill main={drill_main}  "
          f"image lines identical: {rec['image_lines_identical']}")

    json.dump(rec, open(f"{OUT}/result.json", "w"), indent=2, ensure_ascii=False)
    open(f"{OUT}/log.txt", "w").write("\n".join(w.LOG) + "\n")
    w.say(f"  teardown written -> {OUT}/result.json")


if __name__ == "__main__":
    main()