#!/usr/bin/env python3 """Part B — R-659 on controller v0.268.0: chaos round 11 reproduced. nextcloud (declared drive files), a failing update (34.0.1 -> 34.0.4 + a probe port it does not answer), the undo copy CUT OFF during `verifying` (its finished-marker removed — `09` §6.1a's measured case) -> HOLD. No off-site tier on 9202. Expected: the no-whole-copy sentence in hu and en, no Mentések button beside it, the operator event (DROPPED at WARN on this hub-less box — R-620), and NO app_start_failed after the hold (R-660).""" import html as H, json, re, sys, time sys.path.insert(0, ".") import walk as w import fixtures as fx F, SUB, APP = fx.Nextcloud(), "cloud-lb", "nextcloud" CAT = f"{w.DRILL}/templates/{APP}" out = {"controller": w.guest("cat /etc/felhom-controller-image").strip()} w.login() def drill(edit, msg): import fcntl with open(f"{w.SC}/drill.lock", "w") as lk: fcntl.flock(lk, fcntl.LOCK_EX) w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) edit() w.sh(["git", "-C", w.DRILL, "commit", "-qam", "LIVE-B " + msg]) r = w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) h = w.sh(["git", "-C", w.DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip() w.say(f"drill {h}: {msg} push rc={r.returncode}") return h HEAD = open(f"{CAT}/docker-compose.yml").read() STEP1 = open(f"{CAT}/steps/eef2e4afe1218021.yml").read() FY = open(f"{CAT}/.felhom.yml").read() def set_compose(body): return lambda: open(f"{CAT}/docker-compose.yml", "w").write(body) def set_probe(port): def _e(): f = open(f"{CAT}/.felhom.yml").read() open(f"{CAT}/.felhom.yml", "w").write(re.sub(r"(healthcheck:\n\s+checks:\n\s+- type: api\n\s+port: )\d+", lambda m: m.group(1) + str(port), f, count=1)) return _e out["drill_1"] = drill(set_compose(STEP1), "nextcloud template = ladder step 1's own definition (34.0.1 / mariadb 12.3)") w.sync_rescan() out["deployed"] = w.deploy(APP, SUB) A = F.seed(w, SUB, w.say) out["seed_A"] = bool(A) out["drill_2"] = drill(lambda: (set_compose(HEAD)(), set_probe(8999)()), "nextcloud back to the head (34.0.4) + probe port 8999") out["badge_wait"] = w.sync_rescan(APP, "nextcloud:34.0.4-apache", tries=60) h = H.unescape(w.page(f"/apps/{APP}")) out["steps_left_line_hu"] = re.findall(r'data-ladder-steps="(\d+)">([^<]*)<', h) out["steps_left_line_en"] = re.findall(r'data-ladder-steps="(\d+)">([^<]*)<', H.unescape(w.page(f"/apps/{APP}?lang=en"))) w.say(f"steps-left line hu={out['steps_left_line_hu']} en={out['steps_left_line_en']}") cut = {} def on_phase(ph): if ph == "verifying" and not cut: cut["out"] = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={APP} | head -1); echo "copy: $c" docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c | head -5'""") w.say(" >>> finished-marker removed from one undo copy:\n" + cut["out"]) t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) code, d = w.ctl("POST", f"/api/stacks/{APP}/update") w.say(f"Update -> {code}") phases, seen, tp = [], None, time.time() while time.time() - tp < 1500: st = w.stack(APP) ph = st.get("update_phase") if ph != seen and ph: seen = ph phases.append((round(time.time() - tp, 1), ph)) w.say(f" +{phases[-1][0]:6.1f}s phase={ph}") on_phase(ph) if not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - tp > 3: break time.sleep(0.5) held_at = time.time() out["phases"], out["cutoff"] = phases, cut.get("out") st = w.stack(APP) out["api"] = {k: st.get(k) for k in ("state", "update_phase", "hold_reason", "hold_no_whole_copy", "update_error")} w.say("API: " + json.dumps(out["api"], ensure_ascii=False)) for lang in ("hu", "en"): page = H.unescape(w.page(f"/apps/{APP}?lang={lang}")) m = re.search(r'data-held="true">(.*?)', page, re.S) out[f"app_page_{lang}"] = m.group(1).strip() if m else None out[f"app_page_{lang}_restore_button"] = bool(m and 'href="/backups/apps"' in m.group(1)) lst = H.unescape(w.page(f"/stacks?lang={lang}")) m2 = re.search(r'data-held="true">(.*?)', lst, re.S) out[f"stacks_page_{lang}_restore_button"] = bool(m2 and 'href="/backups/apps"' in m2.group(1)) w.say(f"[{lang}] app page held block: {out[f'app_page_{lang}']!r} restore-button app={out[f'app_page_{lang}_restore_button']} list={out[f'stacks_page_{lang}_restore_button']}") # ASCII-fragment checks, with controls (the ui-hungarian rule) hu = out["app_page_hu"] or "" out["ascii_checks"] = {"hu has 'gyfelszolgalat' (positive)": "gyfélszolgálat" in hu, "hu has 'Visszaallithato a Mentesek' (must be absent)": "Visszaállítható a Mentések" in hu, "en has 'Felhom support has been told' (positive)": "Felhom support has been told" in (out["app_page_en"] or "")} w.say("checks: " + json.dumps(out["ascii_checks"])) # R-660: two and a half minutes of the dead-app scan after the hold while time.time() - held_at < 150: time.sleep(10) out["log"] = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'nextcloud|DROPPED|dropped event' | grep -E 'NO copy|DROPPED|dropped event|HELD|UNDO|app_start_failed|no_whole' | tail -30") w.say("controller log:\n" + out["log"]) c2, dl = w.ctl("GET", "/api/debug/logs?level=DEBUG&lines=4000") ents = ((dl.get("data") or {}).get("entries") or []) if isinstance(dl, dict) else [] out["debug_ring_events"] = [e.get("timestamp", "") + " " + e.get("message", "") for e in ents if ("app_start_failed" in e.get("message", "") or "app_hold_no_whole_copy" in e.get("message", "") or "app_update_held" in e.get("message", "")) and e.get("timestamp", "") >= t0] w.say("debug ring (events since the press):\n " + "\n ".join(out["debug_ring_events"])) json.dump(out, open("../partB/01-round11-reproduced.json", "w"), indent=2, ensure_ascii=False, default=str) open("../partB/01-round11-reproduced.log", "w").write("\n".join(w.LOG) + "\n")