admin
86c4b9a0d8
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
118 lines
6.0 KiB
Python
118 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Part B — R-659 on controller v0.268.0: chaos round 11 reproduced. nextcloud (declared drive files),
|
|
a failing update (34.0.1 -> 34.0.4 + a probe port it does not answer), the undo copy CUT OFF during
|
|
`verifying` (its finished-marker removed — `09` §6.1a's measured case) -> HOLD. No off-site tier on 9202.
|
|
Expected: the no-whole-copy sentence in hu and en, no Mentések button beside it, the operator event
|
|
(DROPPED at WARN on this hub-less box — R-620), and NO app_start_failed after the hold (R-660)."""
|
|
import html as H, json, re, sys, time
|
|
sys.path.insert(0, ".")
|
|
import walk as w
|
|
import fixtures as fx
|
|
|
|
F, SUB, APP = fx.Nextcloud(), "cloud-lb", "nextcloud"
|
|
CAT = f"{w.DRILL}/templates/{APP}"
|
|
out = {"controller": w.guest("cat /etc/felhom-controller-image").strip()}
|
|
w.login()
|
|
|
|
|
|
def drill(edit, msg):
|
|
import fcntl
|
|
with open(f"{w.SC}/drill.lock", "w") as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
|
edit()
|
|
w.sh(["git", "-C", w.DRILL, "commit", "-qam", "LIVE-B " + msg])
|
|
r = w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120)
|
|
h = w.sh(["git", "-C", w.DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip()
|
|
w.say(f"drill {h}: {msg} push rc={r.returncode}")
|
|
return h
|
|
|
|
|
|
HEAD = open(f"{CAT}/docker-compose.yml").read()
|
|
STEP1 = open(f"{CAT}/steps/eef2e4afe1218021.yml").read()
|
|
FY = open(f"{CAT}/.felhom.yml").read()
|
|
|
|
|
|
def set_compose(body):
|
|
return lambda: open(f"{CAT}/docker-compose.yml", "w").write(body)
|
|
|
|
|
|
def set_probe(port):
|
|
def _e():
|
|
f = open(f"{CAT}/.felhom.yml").read()
|
|
open(f"{CAT}/.felhom.yml", "w").write(re.sub(r"(healthcheck:\n\s+checks:\n\s+- type: api\n\s+port: )\d+", lambda m: m.group(1) + str(port), f, count=1))
|
|
return _e
|
|
|
|
|
|
out["drill_1"] = drill(set_compose(STEP1), "nextcloud template = ladder step 1's own definition (34.0.1 / mariadb 12.3)")
|
|
w.sync_rescan()
|
|
out["deployed"] = w.deploy(APP, SUB)
|
|
A = F.seed(w, SUB, w.say)
|
|
out["seed_A"] = bool(A)
|
|
out["drill_2"] = drill(lambda: (set_compose(HEAD)(), set_probe(8999)()), "nextcloud back to the head (34.0.4) + probe port 8999")
|
|
out["badge_wait"] = w.sync_rescan(APP, "nextcloud:34.0.4-apache", tries=60)
|
|
h = H.unescape(w.page(f"/apps/{APP}"))
|
|
out["steps_left_line_hu"] = re.findall(r'data-ladder-steps="(\d+)">([^<]*)<', h)
|
|
out["steps_left_line_en"] = re.findall(r'data-ladder-steps="(\d+)">([^<]*)<', H.unescape(w.page(f"/apps/{APP}?lang=en")))
|
|
w.say(f"steps-left line hu={out['steps_left_line_hu']} en={out['steps_left_line_en']}")
|
|
|
|
cut = {}
|
|
|
|
|
|
def on_phase(ph):
|
|
if ph == "verifying" and not cut:
|
|
cut["out"] = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={APP} | head -1); echo "copy: $c"
|
|
docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c | head -5'""")
|
|
w.say(" >>> finished-marker removed from one undo copy:\n" + cut["out"])
|
|
|
|
|
|
t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
code, d = w.ctl("POST", f"/api/stacks/{APP}/update")
|
|
w.say(f"Update -> {code}")
|
|
phases, seen, tp = [], None, time.time()
|
|
while time.time() - tp < 1500:
|
|
st = w.stack(APP)
|
|
ph = st.get("update_phase")
|
|
if ph != seen and ph:
|
|
seen = ph
|
|
phases.append((round(time.time() - tp, 1), ph))
|
|
w.say(f" +{phases[-1][0]:6.1f}s phase={ph}")
|
|
on_phase(ph)
|
|
if not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - tp > 3:
|
|
break
|
|
time.sleep(0.5)
|
|
held_at = time.time()
|
|
out["phases"], out["cutoff"] = phases, cut.get("out")
|
|
st = w.stack(APP)
|
|
out["api"] = {k: st.get(k) for k in ("state", "update_phase", "hold_reason", "hold_no_whole_copy", "update_error")}
|
|
w.say("API: " + json.dumps(out["api"], ensure_ascii=False))
|
|
for lang in ("hu", "en"):
|
|
page = H.unescape(w.page(f"/apps/{APP}?lang={lang}"))
|
|
m = re.search(r'data-held="true">(.*?)</div>', page, re.S)
|
|
out[f"app_page_{lang}"] = m.group(1).strip() if m else None
|
|
out[f"app_page_{lang}_restore_button"] = bool(m and 'href="/backups/apps"' in m.group(1))
|
|
lst = H.unescape(w.page(f"/stacks?lang={lang}"))
|
|
m2 = re.search(r'data-held="true">(.*?)</div>', lst, re.S)
|
|
out[f"stacks_page_{lang}_restore_button"] = bool(m2 and 'href="/backups/apps"' in m2.group(1))
|
|
w.say(f"[{lang}] app page held block: {out[f'app_page_{lang}']!r} restore-button app={out[f'app_page_{lang}_restore_button']} list={out[f'stacks_page_{lang}_restore_button']}")
|
|
# ASCII-fragment checks, with controls (the ui-hungarian rule)
|
|
hu = out["app_page_hu"] or ""
|
|
out["ascii_checks"] = {"hu has 'gyfelszolgalat' (positive)": "gyfélszolgálat" in hu,
|
|
"hu has 'Visszaallithato a Mentesek' (must be absent)": "Visszaállítható a Mentések" in hu,
|
|
"en has 'Felhom support has been told' (positive)": "Felhom support has been told" in (out["app_page_en"] or "")}
|
|
w.say("checks: " + json.dumps(out["ascii_checks"]))
|
|
|
|
# R-660: two and a half minutes of the dead-app scan after the hold
|
|
while time.time() - held_at < 150:
|
|
time.sleep(10)
|
|
out["log"] = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'nextcloud|DROPPED|dropped event' | grep -E 'NO copy|DROPPED|dropped event|HELD|UNDO|app_start_failed|no_whole' | tail -30")
|
|
w.say("controller log:\n" + out["log"])
|
|
c2, dl = w.ctl("GET", "/api/debug/logs?level=DEBUG&lines=4000")
|
|
ents = ((dl.get("data") or {}).get("entries") or []) if isinstance(dl, dict) else []
|
|
out["debug_ring_events"] = [e.get("timestamp", "") + " " + e.get("message", "") for e in ents
|
|
if ("app_start_failed" in e.get("message", "") or "app_hold_no_whole_copy" in e.get("message", "")
|
|
or "app_update_held" in e.get("message", "")) and e.get("timestamp", "") >= t0]
|
|
w.say("debug ring (events since the press):\n " + "\n ".join(out["debug_ring_events"]))
|
|
json.dump(out, open("../partB/01-round11-reproduced.json", "w"), indent=2, ensure_ascii=False, default=str)
|
|
open("../partB/01-round11-reproduced.log", "w").write("\n".join(w.LOG) + "\n")
|