felhom.eu/documentation/audits/security-2026-11-02/part3/proof1102.py

#!/usr/bin/env python3
"""proof1102.py <phase> — live proofs of R-1347, R-1348, R-1351, R-1339 and Traefik v3.7.17 on scratch 9202 (2026-11-02 night).

Front doors only where a household has one (deploy, Update, the restore button, Start, Remove). The faults are real docker
state, not switches: the undo copy loses its finished-marker (R-1347), and a created-only container holds the app's volume so the
crash put-back (R-1348) cannot replace it. The cut is `docker kill` of the controller at the log line „the restore copy is recorded".

Phases, in order:
  boot-before   guest reboot on the controller as found; the FileBrowser lines of that boot
  switch        load the local-only test image (never pushed), point the box at it and at the event sink
  boot-after    guest reboot on the test image; the same lines; Traefik's and the file manager's state
  r1347         radicale: an update that fails with an undo copy cut off → HOLD; the household's restore
                lifts the hold and forgets the flag; 7 kill -9 → the crash-loop watcher stops the app (it watches again)
  r1348         the household's Start, then a restore cut at the copy's record, the volume held, the controller started
                → the put-back fails at start-up → the hold AND the operator event in the sink
  teardown      blocker gone, app removed through the product, controller back on the image it had, yaml byte-identical

Env: SC (0600 scratch with .ctlpw), EV (evidence dir). No secret is printed; the sink stores event bodies only."""
import json, os, subprocess, sys, threading, time

sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
os.environ.setdefault("TARGET", "9202")
import box_walk as w
import upgrade_fixtures_box as fb

EV = os.environ.get("EV", ".")
APP, SUB = "radicale", getattr(fb.FIXTURES["radicale"], "sub", "radicale")
VOL = "/var/lib/docker/volumes/felhom-controller-data/_data"
YAML = f"{VOL}/controller.yaml"
SAVE = f"{VOL}/controller.yaml.pre-proof1102"
IMG_FOUND = "gitea.dooplex.hu/admin/felhom-controller:0.349.0-rc.1"
IMG_TEST = "gitea.dooplex.hu/admin/felhom-controller:0.350.0-rc.1-sec1102fault"
SINK_PORT = 18091
SINK_FILE = "/tmp/sec1102-sink.jsonl"
BLOCKER = "sec1102-blocker"
MARKER = 'felhom-undo-complete'
DRILL = w.DRILL
STATE = os.path.join(os.environ["SC"], ".proof-state.json")


def say(*a):
    w.say(*a)


def flush(name):
    with open(os.path.join(EV, f"{name}.log"), "a") as f:
        f.write("\n".join(w.LOG) + "\n")


def guest(script, timeout=600):
    return w.guest(script, timeout=timeout)


def ctl_log(since, rx):
    return guest(f"docker logs --since {since} felhom-controller 2>&1 | grep -E '{rx}' | grep -v DEBUG | cut -c1-260 | tail -40")


def now():
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())


def wait_login(tries=60):
    for _ in range(tries):
        time.sleep(5)
        try:
            w.login()
            return True
        except SystemExit:
            continue
    return False


def boot_lines(tag):
    """Reboot the guest and read this boot's FileBrowser and first-boot lines from the controller's log."""
    t0 = now()
    r = w.sh(w.ssh_args() + [f"pct reboot {w.GUEST}"], timeout=180)
    say(f"  pct reboot {w.GUEST} rc={r.returncode}")
    time.sleep(20)
    ok = wait_login(40)
    say(f"  login after the reboot: {ok}")
    time.sleep(45)   # the base stack's round and the mounts sync
    img = guest("cat /etc/felhom-controller-image; docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep -E 'felhom-controller|traefik|filebrowser|cloudflared'")
    say(f"  images:\n{img}")
    lines = guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'Conflict|first-boot|filebrowser compose exists|FileBrowser mounts synced|FileBrowser sync|not mounted|skipping its folder|could not leave|traefik' | grep -v DEBUG | cut -c1-240")
    say(f"  [{tag}] FileBrowser lines of this boot:\n{lines}")
    say(f"  [{tag}] Conflict lines: {lines.count('Conflict')}; first-boot WARN: {lines.count('first-boot base-stack')}; 'not mounted' lines: {lines.count('not mounted')}")


def phase_boot_before():
    w.login()
    boot_lines("before")


def start_sink():
    """The event sink: a tiny HTTP server in the guest, event bodies only. hub.enabled stays false, so nothing else moves. /tmp is a
    tmpfs here, so a guest reboot takes file and process away: it is written again each time."""
    sink = f"""
import http.server, json
class H(http.server.BaseHTTPRequestHandler):
    def do_POST(self):
        n = int(self.headers.get('Content-Length') or 0); b = self.rfile.read(n)
        if self.path.endswith('/event'):
            open({SINK_FILE!r}, 'ab').write(b + b'\\n')
        self.send_response(200); self.send_header('Content-Type', 'application/json'); self.end_headers(); self.wfile.write(b'{{}}')
    def do_GET(self):
        self.send_response(404); self.end_headers()
    def log_message(self, *a): pass
http.server.HTTPServer(('0.0.0.0', {SINK_PORT}), H).serve_forever()
"""
    return guest(f"cat > /tmp/sec1102-sink.py <<'EOF'\n{sink}\nEOF\nrm -f {SINK_FILE}; pgrep -f 'sec1102-sin[k]' >/dev/null || (nohup python3 /tmp/sec1102-sink.py >/dev/null 2>&1 &)\nsleep 1; curl -s -o /dev/null -w '%{{http_code}}' -X POST http://127.0.0.1:{SINK_PORT}/ping; echo")


def phase_switch():
    say(guest("docker ps --format '{{.Names}} {{.Image}} {{.Status}}'; cat /etc/felhom-controller-image; "
              "python3 - <<'PY'\nimport json\ns=json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json'))\n"
              "print('holds as found:', sorted((s.get('restore_holds') or {}).keys()))\nPY"))
    tar = os.environ["IMG_TAR"]
    w.sh(["scp", "-q", tar, f"{w._T['ssh']}:/tmp/sec1102-ctl.tar.gz"], timeout=300)
    w.sh(w.ssh_args() + [f"pct push {w.GUEST} /tmp/sec1102-ctl.tar.gz /tmp/sec1102-ctl.tar.gz && rm -f /tmp/sec1102-ctl.tar.gz"], timeout=300)
    say(guest("zcat /tmp/sec1102-ctl.tar.gz | docker load | tail -1; rm -f /tmp/sec1102-ctl.tar.gz", timeout=400))
    start_sink()
    ip = guest("ip -4 -o addr show eth0 | awk '{print $4}' | cut -d/ -f1").strip()
    say(f"  sink on {ip}:{SINK_PORT}")
    say(guest(f"""set -e
test -f {SAVE} || cp -p {YAML} {SAVE}
python3 - <<'PY'
import re
p = {YAML!r}; s = open(p).read()
s = re.sub(r'(^hub:\\n\\s+api_key: )""', r'\\g<1>"sec1102-test"', s, count=1, flags=re.M)
s = re.sub(r'(^hub:(?:\\n\\s+.*)*?\\n\\s+url: )""', r'\\g<1>"http://{ip}:{SINK_PORT}"', s, count=1, flags=re.M)
open(p, 'w').write(s)
PY
sed -n '/^hub:/,/url:/p' {YAML} | sed -E 's/api_key: .*/api_key: (set)/'
echo {IMG_TEST} > /etc/felhom-controller-image
systemctl restart felhom-controller-bootstrap.service
sleep 8; docker ps --format '{{{{.Names}}}} {{{{.Image}}}} {{{{.Status}}}}' | grep felhom-controller"""))
    say(f"  login: {wait_login(30)}")
    say(guest("docker logs felhom-controller 2>&1 | grep -E 'Notifier enabled|Notifier disabled' | tail -2 | cut -c1-200; docker exec felhom-controller felhom-controller --version | cut -c1-120"))


def phase_boot_stage():
    """The race needs a boot where the sync RECREATES: both boots above found nothing changed, so no race (0 Conflict before too).
    A comment appended to FileBrowser's compose makes the file differ from what the sync renders, as after an update."""
    w.login()
    # Stage 2 (STOP_FB=1): the file manager also stays down through the boot, so the base stack's bring-up starts it too: both paths.
    stop = "docker stop filebrowser >/dev/null; " if os.environ.get("STOP_FB") else ""
    say(guest(stop + "echo '# sec1102 stage' >> /opt/docker/stacks/filebrowser/docker-compose.yml; tail -1 /opt/docker/stacks/filebrowser/docker-compose.yml; cat /etc/felhom-controller-image"))
    boot_lines(os.environ.get("BOOT_TAG", "stage"))


def phase_boot_after():
    w.login()
    boot_lines("after")
    code, d = w.ctl("GET", "/api/health")
    say(f"  /api/health {code}")


def phase_r1347():
    w.login()
    drill_imgs = w.images_of(open(f"{DRILL}/templates/{APP}/docker-compose.yml").read())
    say(f"  drill's {APP}: {drill_imgs}")
    svc, frm = next(iter(drill_imgs.items()))
    if w.stack(APP).get("deployed"):
        w.remove(APP)
    w.refresh_box_mirror()
    w.sync_rescan()
    assert w.deploy(APP, SUB), "the install did not complete"
    w.wait_app(SUB, "/", tries=60)
    fx = fb.FIXTURES[APP]
    tok = fx.seed(w, SUB, say)
    say(f"  seeded: {bool(tok)}; read back: {tok is not None and fx.verify(w, SUB, tok, say)}")
    json.dump({"tok": tok, "generated": getattr(w, "GENERATED", {})}, open(STATE, "w"))
    os.chmod(STATE, 0o600)
    vols = guest(f"docker volume ls --format '{{{{.Name}}}}' | grep '^{APP}_'").split()
    say(f"  volumes: {vols}")
    guest(f"docker rm -f {BLOCKER} 2>/dev/null")
    commit = w.drill_bump(APP, frm, "busybox:1.36.1")
    say(f"  drill commit {commit}")
    say(f"  box read TO after {w.sync_rescan(APP, 'busybox:1.36.1', tries=40, delay=10)} s")
    t0 = now()
    # The undo must fail for a hold to exist. A volume held by a created-only container does not stop it (tried first: the undo
    # empties the volume and pours), so the undo copy is cut off, the shape of a helper killed mid-copy: its finished-marker goes
    # while the update verifies, and the undo refuses to pour („NOTHING is put back", TestUndo_CutOffCopyIsRefusedBeforeAnythingMoves).
    def cut_copy():
        for _ in range(120):
            time.sleep(3)
            names = guest(f"docker volume ls --format '{{{{.Name}}}}' | grep '^{APP}_.*\\.pre-update-'").split()
            if names:
                time.sleep(15)
                say("  the undo copy cut off: " + names[0] + " " + guest(f"docker run --rm -v {names[0]}:/c busybox:1.36.1 sh -c 'rm -f /c/{MARKER} && ls -a /c | tr \"\\n\" \" \"'"))
                return
    cutter = threading.Thread(target=cut_copy)
    cutter.start()
    u = w.press_update(APP, poll=2.0, cap_s=1500)
    cutter.join(30)
    say(f"  update: accepted={u.get('accepted')} final={u.get('final_phase')} err={str(u.get('update_error'))[:200]}")
    time.sleep(10)
    st = w.stack(APP)
    say(f"  after the update: state={st.get('state')} hold={st.get('hold_reason')!r} phase={st.get('update_phase')} key={st.get('update_error_key')}")
    say("  controller log:\n" + ctl_log(t0, f"update {APP}|{APP}: .*(HOLD|hold|undo)|restore hold|Failed to remove"))
    flush("r1347-hold")
    return phase_r1347b()


def hold_code():
    """The hold's own code, from the box's settings: the API's hold_reason is the household's sentence."""
    return guest(f"python3 -c \"import json;s=json.load(open('{VOL}/data/settings.json'));print((s.get('restore_holds') or {{}}).get('{APP}',{{}}).get('reason',''))\"").strip()


def phase_r1347b():
    w.login()
    code = hold_code()
    say(f"  the hold in the box's settings: {code!r}")
    if code != "update_failed":
        say("  STOP: no update hold — the stage did not work; nothing to prove")
        return 1
    fx = fb.FIXTURES[APP]
    tok = None
    try:
        tok = (json.load(open(STATE)) or {}).get("tok")
    except (OSError, ValueError):
        pass
    t0 = now()
    res = w.restore(APP)
    say(f"  restore: {json.dumps({k: res.get(k) for k in ('ok', 'snapshot_id', 'http', 'seconds', 'state_after', 'hold_after')})}")
    time.sleep(10)
    st = w.stack(APP)
    say(f"  after the restore: state={st.get('state')} hold={st.get('hold_reason')!r} phase={st.get('update_phase')} update_error={st.get('update_error')!r} key={st.get('update_error_key')!r}")
    say("  controller log:\n" + ctl_log(t0, "update hold|R-1347|update-held|undo cop|restore completed"))
    say(f"  the seed reads back: {tok is not None and fx.verify(w, SUB, tok, say)}")
    subprocess.run(["git", "-C", DRILL, "fetch", "-q", "live"], check=False)
    subprocess.run(["git", "-C", DRILL, "reset", "-q", "--hard", "live/main"], check=False)
    subprocess.run(["git", "-C", DRILL, "push", "-q", "-f", "origin", "main"], check=False)
    w.refresh_box_mirror()
    w.sync_rescan()
    # the consequence: the crash-loop watcher watches this app again (D53's recipe: seven kill -9, 15 s apart)
    t0 = now()
    for i in range(10):
        pid = guest(f"docker inspect -f '{{{{.State.Pid}}}}' {APP} 2>/dev/null").strip()
        if pid and pid != "0":
            guest(f"kill -9 {pid}")
        say(f"  kill -9 #{i + 1} pid {pid}; state {w.stack(APP).get('state')} hold {w.stack(APP).get('hold_reason')!r}")
        if w.stack(APP).get("hold_reason"):
            break
        time.sleep(15)
    time.sleep(35)
    st = w.stack(APP)
    code = hold_code()
    say(f"  after the kills: state={st.get('state')} hold code={code!r}")
    say("  controller log:\n" + ctl_log(t0, "crash_loop|STOPPING|unhealthy"))
    return 0 if code == "unhealthy_stop" else 1


def phase_r1348():
    w.login()
    st0 = json.load(open(STATE))
    if getattr(w, "GENERATED", None) is not None:
        w.GENERATED.update(st0.get("generated") or {})
    st = w.stack(APP)
    say(f"  before: state={st.get('state')} hold={st.get('hold_reason')!r}")
    if hold_code() == "unhealthy_stop":
        c, d = w.ctl("POST", f"/api/stacks/{APP}/start")   # the household's Start lifts this one (D53)
        say(f"  Start -> {c} {str(d)[:120]}")
        for _ in range(30):
            time.sleep(5)
            st = w.stack(APP)
            if st.get("state") == "running":
                break
        say(f"  after Start: state={st.get('state')} hold={st.get('hold_reason')!r}")
    say("  sink answers: " + start_sink())
    vols = guest(f"docker volume ls --format '{{{{.Name}}}}' | grep '^{APP}_'").split()
    # The cut, inside the guest so it has no ssh delay: at the log line, kill the controller, hold the volume, start again.
    cut = (f"timeout 400 sh -c \"docker logs -f --since 2s felhom-controller 2>&1 | grep -m1 'the restore copy is recorded' > /tmp/sec1102-cut.txt\"; "
           f"docker kill felhom-controller >/dev/null; docker create --name {BLOCKER} -v {vols[0]}:/x busybox:1.36.1 true >/dev/null; "
           f"sleep 2; docker start felhom-controller >/dev/null; cut -c1-200 /tmp/sec1102-cut.txt; date -u +%H:%M:%S; docker ps -a --format '{{{{.Names}}}} {{{{.Status}}}}' | grep -E 'felhom-controller|{BLOCKER}'")
    out = {}
    th = threading.Thread(target=lambda: out.update(cut=guest(cut, timeout=450)))
    t0 = now()
    th.start()
    time.sleep(4)
    code, snaps = w.ctl("GET", f"/api/backup/snapshots?stack={APP}")
    snap = w.snapshots(APP)[0]
    sid = snap.get("id") or snap.get("snapshot_id") or snap.get("short_id")
    sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
    csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
    r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
              "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={APP}", "--data-urlencode", f"snapshot_id={sid}",
              f"{w.BASE}/backup/restore"], timeout=120)
    say(f"  restore pressed: {(r.stdout or '').splitlines()[:1]} snapshot {sid}")
    th.join(460)
    say(f"  the cut:\n{out.get('cut')}")
    say(f"  login after the cut: {wait_login(40)}")
    time.sleep(30)
    st = w.stack(APP)
    code = hold_code()
    say(f"  after the start: state={st.get('state')} hold code={code!r}")
    say("  controller log:\n" + ctl_log(t0, "crash recovery|could NOT be put back|restore hold|HELD|Notifier enabled"))
    events = guest(f"cat {SINK_FILE} 2>/dev/null | cut -c1-1400")
    say(f"  the sink's events:\n{events}")
    mailed = [json.loads(l) for l in events.splitlines() if l.strip().startswith("{") and "put back" in l]
    say(f"  R-1348: operator events with the sentence: {len(mailed)}")
    return 0 if code == "restore_mixed" and mailed else 1


def phase_teardown():
    w.login()
    say("  blocker: " + guest(f"docker rm -f {BLOCKER} 2>&1 | tail -1"))
    st = w.stack(APP)
    say(f"  {APP}: deployed={st.get('deployed')} state={st.get('state')} hold={st.get('hold_reason')!r}")
    if st.get("deployed"):
        w.remove(APP)
    say(guest(f"""cp -p {SAVE} {YAML} && cmp {SAVE} {YAML} && echo YAML-RESTORED-IDENTICAL; rm -f {SAVE}
pkill -f 'sec1102-sin[k]' || true; rm -f /tmp/sec1102-sink.py /tmp/sec1102-cut.txt
cp {SINK_FILE} /tmp/sec1102-sink-keep.jsonl 2>/dev/null; rm -f {SINK_FILE}
echo {IMG_FOUND} > /etc/felhom-controller-image
systemctl restart felhom-controller-bootstrap.service; sleep 10
docker image rm {IMG_TEST} 2>&1 | tail -1 | cut -c1-80
docker ps --format '{{{{.Names}} }} {{{{.Image}}}} {{{{.Status}}}}'
ls /opt/docker/stacks/{APP}/app.yaml 2>&1 | tail -1; docker volume ls --format '{{{{.Name}}}}' | grep '^{APP}_' || echo no-volume
python3 - <<'PY'
import json
s=json.load(open('{VOL}/data/settings.json'))
print('holds left:', sorted((s.get('restore_holds') or {{}}).keys()))
PY"""))


def main():
    ph = sys.argv[1]
    rc = {"boot-before": phase_boot_before, "switch": phase_switch, "boot-after": phase_boot_after, "boot-stage": phase_boot_stage, "r1347": phase_r1347, "r1347b": phase_r1347b,
          "r1348": phase_r1348, "teardown": phase_teardown}[ph]() or 0
    flush(ph)
    say(f"PHASE {ph} rc={rc}")
    return rc


if __name__ == "__main__":
    sys.exit(main())