admin
1cedf94d6b
gates / gates (push) Successful in 29s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
118 lines
6.6 KiB
Python
118 lines
6.6 KiB
Python
#!/usr/bin/env python3
|
|
"""A1 live, whole: (a) a fresh second copy via the update's own backup; (b) through WebDAV: delete file 0, edit
|
|
file 1 (live newer than the copy); (c) a failing update (probe 8999) whose undo copy is cut off → HOLD, the
|
|
page naming the second drive; (d) the second copy's „Teljes visszaállítás" = the WHOLE restore; (e) read back."""
|
|
import html as H, json, re, sys, time
|
|
sys.path.insert(0, ".")
|
|
import walk as w, fixtures as fx, ncfiles as nf
|
|
F, SUB, APP = fx.Nextcloud(), "cloud-a1", "nextcloud"
|
|
CAT = f"{w.DRILL}/templates/{APP}"
|
|
S = json.load(open(f"{w.SC}/a1-secret-state.json"))
|
|
A, files = S["A"], S["files"]
|
|
names = sorted(files)
|
|
out = {"controller": w.guest("cat /etc/felhom-controller-image").strip()}
|
|
w.login()
|
|
|
|
|
|
def drill(edit, msg):
|
|
import fcntl
|
|
with open(f"{w.SC}/drill.lock", "w") as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
|
edit()
|
|
w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-A1b " + msg])
|
|
w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120)
|
|
w.say("drill: " + msg)
|
|
|
|
|
|
HEAD = open(f"{CAT}/docker-compose.yml").read()
|
|
FY = open(f"{CAT}/.felhom.yml").read()
|
|
# (a)
|
|
drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(HEAD.replace("nextcloud:34.0.4-apache", "nextcloud:34.0.97-notag")), "notag")
|
|
w.sync_rescan(APP, "nextcloud:34.0.97-notag", tries=60)
|
|
out["press_a"] = w.press_update(APP)
|
|
out["log_a"] = w.guest("docker logs --since 3m felhom-controller 2>&1 | grep -iE 'nextcloud' | grep -iE 'Tier 2|precondition|same disk|same-disk' | tail -6")
|
|
w.say("log (a):\n" + out["log_a"])
|
|
drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(HEAD), "back to 34.0.4")
|
|
out["record"] = w.guest("python3 -c \"import json;d=json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json'));print(json.dumps(d['app_backup']['nextcloud']['cross_drive']))\"")
|
|
w.say("Tier-2 record: " + out["record"])
|
|
dest = json.loads(out["record"]).get("destination_path", "")
|
|
out["mirror_seeds"] = w.guest(f"find {dest}/backups/secondary/nextcloud -name 'felhom-seed-*' | sed 's|.*/||'; stat -c '%n dev=%d' {dest} /mnt/felhom-drives/scratch_hdd/userdata/nextcloud")
|
|
w.say("mirror seeds + devices:\n" + out["mirror_seeds"])
|
|
# (b)
|
|
time.sleep(2)
|
|
rc, code, _ = w.app_curl(SUB, f"/remote.php/dav/files/{A['uid']}/{names[0]}", "-u", f"{A['uid']}:{A['pw']}", method="DELETE")
|
|
w.say(f"(b) DELETE {names[0]} -> {code}")
|
|
files1 = "edited by the household " + str(time.time())
|
|
ok = nf.put(w, SUB, A, names[1], files1)
|
|
w.say(f"(b) PUT {names[1]} (newer than the copy) -> {ok}")
|
|
out["after_b"] = nf.verify_files(w, SUB, A, {names[0]: files[names[0]], names[1]: files1, names[2]: files[names[2]]}, w.say)
|
|
# (c)
|
|
def break_edge():
|
|
open(f"{CAT}/docker-compose.yml", "w").write(HEAD.replace("image: redis:7-alpine", "image: redis:7.2-alpine"))
|
|
f = re.sub(r"(healthcheck:\n\s+checks:\n\s+- type: api\n\s+port: )80\b", r"\g<1>8999", FY, count=1)
|
|
assert f != FY, "probe port not found"
|
|
open(f"{CAT}/.felhom.yml", "w").write(f)
|
|
|
|
|
|
drill(break_edge, "redis 7-alpine -> 7.2-alpine + probe 8999 (the failing edge)")
|
|
w.sync_rescan(APP, "redis:7.2-alpine", tries=60)
|
|
cut = {}
|
|
code, d = w.ctl("POST", f"/api/stacks/{APP}/update")
|
|
w.say(f"(c) Update -> {code}")
|
|
seen, tp = None, time.time()
|
|
while time.time() - tp < 1200:
|
|
st = w.stack(APP)
|
|
ph = st.get("update_phase")
|
|
if ph != seen and ph:
|
|
seen = ph
|
|
w.say(f" +{time.time()-tp:6.1f}s phase={ph}")
|
|
if ph == "verifying" and not cut:
|
|
cut["out"] = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={APP} | head -1); echo "copy: $c"
|
|
docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c'""")
|
|
w.say(" >>> marker removed:\n" + cut["out"])
|
|
if not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - tp > 3:
|
|
break
|
|
time.sleep(0.5)
|
|
st = w.stack(APP)
|
|
out["held"] = {k: st.get(k) for k in ("update_phase", "hold_reason", "hold_no_whole_copy", "hold_kind", "state")}
|
|
w.say("(c) held: " + json.dumps(out["held"], ensure_ascii=False))
|
|
for lang in ("hu", "en"):
|
|
page = H.unescape(w.page(f"/apps/{APP}?lang={lang}"))
|
|
m = re.search(r'data-held="true">(.*?)</div>', page, re.S)
|
|
out[f"page_{lang}"] = m.group(1).strip() if m else None
|
|
w.say(f"(c) [{lang}] {out[f'page_{lang}']!r}")
|
|
# (d) the WHOLE restore: the button the page offers on the Mentések page for the second copy
|
|
drill(lambda: (open(f"{CAT}/docker-compose.yml", "w").write(HEAD), open(f"{CAT}/.felhom.yml", "w").write(FY)), "back to the head + the real probe")
|
|
w.sync_rescan()
|
|
sess = open(f"{w.SC}/sess{__import__('os').getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{__import__('os').getpid()}.txt").read().strip()
|
|
t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST",
|
|
"--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={APP}", f"{w.BASE}/backup/tier2/unit-restore"], timeout=120)
|
|
w.say("(d) POST /backup/tier2/unit-restore -> " + (r.stdout or "").split("\n")[0] + " " + str([l for l in (r.stdout or "").split("\n") if l.lower().startswith("location")]))
|
|
last = None
|
|
for _ in range(600):
|
|
c2, dd = w.ctl("GET", "/api/backup/restore-status")
|
|
d2 = (dd.get("data") or {}) if isinstance(dd, dict) else {}
|
|
cur = (d2.get("running"), d2.get("message"))
|
|
if cur != last:
|
|
w.say(f" restore-status {cur}")
|
|
last = cur
|
|
if not d2.get("running") and d2.get("message"):
|
|
break
|
|
time.sleep(2)
|
|
out["restore_msg"] = last
|
|
out["restore_log"] = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'whole restore|WHOLE restore|Tier-2 whole|Restore-from-unit|hold .*CLEARED' | tail -8")
|
|
w.say("(d) log:\n" + out["restore_log"])
|
|
# (e)
|
|
w.wait_app(SUB, "/status.php", want=("200",), tries=120, delay=5)
|
|
out["account"] = bool(F.verify(w, SUB, A, w.say))
|
|
out["files_after"] = nf.verify_files(w, SUB, A, {names[0]: files[names[0]], names[1]: files1, names[2]: files[names[2]]}, w.say)
|
|
out["kept_beside"] = w.guest(f"find /mnt/felhom-drives/scratch_hdd/userdata/nextcloud -name '*.felhom-*' | sed 's|.*/||'")
|
|
st = w.stack(APP)
|
|
out["final"] = {k: st.get(k) for k in ("state", "hold_reason", "update_phase")}
|
|
w.say("(e) final " + json.dumps(out["final"], ensure_ascii=False) + " kept-beside: " + out["kept_beside"])
|
|
json.dump(out, open("../A1/20-whole-restore.json", "w"), indent=2, ensure_ascii=False, default=str)
|
|
open("../A1/20-whole-restore.log", "w").write("\n".join(w.LOG) + "\n")
|