#!/usr/bin/env python3 """A1 live, whole: (a) a fresh second copy via the update's own backup; (b) through WebDAV: delete file 0, edit file 1 (live newer than the copy); (c) a failing update (probe 8999) whose undo copy is cut off → HOLD, the page naming the second drive; (d) the second copy's „Teljes visszaállítás" = the WHOLE restore; (e) read back.""" import html as H, json, re, sys, time sys.path.insert(0, ".") import walk as w, fixtures as fx, ncfiles as nf F, SUB, APP = fx.Nextcloud(), "cloud-a1", "nextcloud" CAT = f"{w.DRILL}/templates/{APP}" S = json.load(open(f"{w.SC}/a1-secret-state.json")) A, files = S["A"], S["files"] names = sorted(files) out = {"controller": w.guest("cat /etc/felhom-controller-image").strip()} w.login() def drill(edit, msg): import fcntl with open(f"{w.SC}/drill.lock", "w") as lk: fcntl.flock(lk, fcntl.LOCK_EX) w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) edit() w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-A1b " + msg]) w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) w.say("drill: " + msg) HEAD = open(f"{CAT}/docker-compose.yml").read() FY = open(f"{CAT}/.felhom.yml").read() # (a) drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(HEAD.replace("nextcloud:34.0.4-apache", "nextcloud:34.0.97-notag")), "notag") w.sync_rescan(APP, "nextcloud:34.0.97-notag", tries=60) out["press_a"] = w.press_update(APP) out["log_a"] = w.guest("docker logs --since 3m felhom-controller 2>&1 | grep -iE 'nextcloud' | grep -iE 'Tier 2|precondition|same disk|same-disk' | tail -6") w.say("log (a):\n" + out["log_a"]) drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(HEAD), "back to 34.0.4") out["record"] = w.guest("python3 -c \"import json;d=json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json'));print(json.dumps(d['app_backup']['nextcloud']['cross_drive']))\"") w.say("Tier-2 record: " + out["record"]) dest = json.loads(out["record"]).get("destination_path", "") out["mirror_seeds"] = w.guest(f"find {dest}/backups/secondary/nextcloud -name 'felhom-seed-*' | sed 's|.*/||'; stat -c '%n dev=%d' {dest} /mnt/felhom-drives/scratch_hdd/userdata/nextcloud") w.say("mirror seeds + devices:\n" + out["mirror_seeds"]) # (b) time.sleep(2) rc, code, _ = w.app_curl(SUB, f"/remote.php/dav/files/{A['uid']}/{names[0]}", "-u", f"{A['uid']}:{A['pw']}", method="DELETE") w.say(f"(b) DELETE {names[0]} -> {code}") files1 = "edited by the household " + str(time.time()) ok = nf.put(w, SUB, A, names[1], files1) w.say(f"(b) PUT {names[1]} (newer than the copy) -> {ok}") out["after_b"] = nf.verify_files(w, SUB, A, {names[0]: files[names[0]], names[1]: files1, names[2]: files[names[2]]}, w.say) # (c) def break_edge(): open(f"{CAT}/docker-compose.yml", "w").write(HEAD.replace("image: redis:7-alpine", "image: redis:7.2-alpine")) f = re.sub(r"(healthcheck:\n\s+checks:\n\s+- type: api\n\s+port: )80\b", r"\g<1>8999", FY, count=1) assert f != FY, "probe port not found" open(f"{CAT}/.felhom.yml", "w").write(f) drill(break_edge, "redis 7-alpine -> 7.2-alpine + probe 8999 (the failing edge)") w.sync_rescan(APP, "redis:7.2-alpine", tries=60) cut = {} code, d = w.ctl("POST", f"/api/stacks/{APP}/update") w.say(f"(c) Update -> {code}") seen, tp = None, time.time() while time.time() - tp < 1200: st = w.stack(APP) ph = st.get("update_phase") if ph != seen and ph: seen = ph w.say(f" +{time.time()-tp:6.1f}s phase={ph}") if ph == "verifying" and not cut: cut["out"] = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={APP} | head -1); echo "copy: $c" docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c'""") w.say(" >>> marker removed:\n" + cut["out"]) if not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - tp > 3: break time.sleep(0.5) st = w.stack(APP) out["held"] = {k: st.get(k) for k in ("update_phase", "hold_reason", "hold_no_whole_copy", "hold_kind", "state")} w.say("(c) held: " + json.dumps(out["held"], ensure_ascii=False)) for lang in ("hu", "en"): page = H.unescape(w.page(f"/apps/{APP}?lang={lang}")) m = re.search(r'data-held="true">(.*?)', page, re.S) out[f"page_{lang}"] = m.group(1).strip() if m else None w.say(f"(c) [{lang}] {out[f'page_{lang}']!r}") # (d) the WHOLE restore: the button the page offers on the Mentések page for the second copy drill(lambda: (open(f"{CAT}/docker-compose.yml", "w").write(HEAD), open(f"{CAT}/.felhom.yml", "w").write(FY)), "back to the head + the real probe") w.sync_rescan() sess = open(f"{w.SC}/sess{__import__('os').getpid()}.txt").read().strip() csrf = open(f"{w.SC}/csrf{__import__('os').getpid()}.txt").read().strip() t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST", "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={APP}", f"{w.BASE}/backup/tier2/unit-restore"], timeout=120) w.say("(d) POST /backup/tier2/unit-restore -> " + (r.stdout or "").split("\n")[0] + " " + str([l for l in (r.stdout or "").split("\n") if l.lower().startswith("location")])) last = None for _ in range(600): c2, dd = w.ctl("GET", "/api/backup/restore-status") d2 = (dd.get("data") or {}) if isinstance(dd, dict) else {} cur = (d2.get("running"), d2.get("message")) if cur != last: w.say(f" restore-status {cur}") last = cur if not d2.get("running") and d2.get("message"): break time.sleep(2) out["restore_msg"] = last out["restore_log"] = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'whole restore|WHOLE restore|Tier-2 whole|Restore-from-unit|hold .*CLEARED' | tail -8") w.say("(d) log:\n" + out["restore_log"]) # (e) w.wait_app(SUB, "/status.php", want=("200",), tries=120, delay=5) out["account"] = bool(F.verify(w, SUB, A, w.say)) out["files_after"] = nf.verify_files(w, SUB, A, {names[0]: files[names[0]], names[1]: files1, names[2]: files[names[2]]}, w.say) out["kept_beside"] = w.guest(f"find /mnt/felhom-drives/scratch_hdd/userdata/nextcloud -name '*.felhom-*' | sed 's|.*/||'") st = w.stack(APP) out["final"] = {k: st.get(k) for k in ("state", "hold_reason", "update_phase")} w.say("(e) final " + json.dumps(out["final"], ensure_ascii=False) + " kept-beside: " + out["kept_beside"]) json.dump(out, open("../A1/20-whole-restore.json", "w"), indent=2, ensure_ascii=False, default=str) open("../A1/20-whole-restore.log", "w").write("\n".join(w.LOG) + "\n")