#!/usr/bin/env python3 """A1 live, whole: (a) a fresh second copy via the update's own backup; (b) through WebDAV: delete file 0, edit file 1 (live newer than the copy); (c) a failing update (probe 8999) whose undo copy is cut off → HOLD, the page naming the second drive; (d) the second copy's „Teljes visszaállítás" = the WHOLE restore; (e) read back.""" import html as H, json, re, sys, time sys.path.insert(0, ".") import walk as w, fixtures as fx, ncfiles as nf F, SUB, APP = fx.Nextcloud(), "cloud-a1", "nextcloud" CAT = f"{w.DRILL}/templates/{APP}" MIR = "/mnt/sys_drive/felhom-data/backups/secondary/nextcloud" S = json.load(open(f"{w.SC}/a1-secret-state.json")) A, files = S["A"], S["files"] names = sorted(files) out = {"controller": w.guest("cat /etc/felhom-controller-image").strip()} w.login() def drill(edit, msg): import fcntl with open(f"{w.SC}/drill.lock", "w") as lk: fcntl.flock(lk, fcntl.LOCK_EX) w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) edit() w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-A2 " + msg]) w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) w.say("drill: " + msg) HEAD = open(f"{CAT}/docker-compose.yml").read() FY = open(f"{CAT}/.felhom.yml").read() names = sorted(files) files1 = None def break_edge(): open(f"{CAT}/docker-compose.yml", "w").write(HEAD.replace("image: redis:7-alpine", "image: redis:7.4.0-alpine")) f = re.sub(r"(healthcheck:\n\s+checks:\n\s+- type: api\n\s+port: )80\b", r"\g<1>8999", FY, count=1) assert f != FY, "probe port not found" open(f"{CAT}/.felhom.yml", "w").write(f) drill(break_edge, "redis 7-alpine -> 7.4.0-alpine + probe 8999 (the failing edge)") w.sync_rescan(APP, "redis:7.4.0-alpine", tries=60) cut = {} code, d = w.ctl("POST", f"/api/stacks/{APP}/update") w.say(f"(c) Update -> {code}") seen, tp = None, time.time() while time.time() - tp < 1200: st = w.stack(APP) ph = st.get("update_phase") if ph != seen and ph: seen = ph w.say(f" +{time.time()-tp:6.1f}s phase={ph}") if ph == "verifying" and not cut: cut["out"] = w.guest(f"""c=$(docker volume ls -q --filter label=felhom.undo-copy-of={APP} | head -1); echo "copy: $c" docker run --rm -v $c:/c alpine sh -c 'rm -f /c/felhom-undo-complete; ls /c' mv {MIR} {MIR}.night-aside && echo "second-drive mirror moved aside (a one-drive box)" """) w.say(" >>> marker removed:\n" + cut["out"]) if not st.get("updating") and ph in ("done", "failed", "undone") and time.time() - tp > 3: break time.sleep(0.5) st = w.stack(APP) out["held"] = {k: st.get(k) for k in ("update_phase", "hold_reason", "hold_no_whole_copy", "hold_kind", "state")} w.say("(c) held: " + json.dumps(out["held"], ensure_ascii=False)) for lang in ("hu", "en"): page = H.unescape(w.page(f"/apps/{APP}?lang={lang}")) m = re.search(r'data-held="true">(.*?)', page, re.S) out[f"page_{lang}"] = m.group(1).strip() if m else None w.say(f"(c) [{lang}] {out[f'page_{lang}']!r}") # decision 27, positive control / refusal: what the Remove dialog reads, and the Remove itself code, hd = w.ctl("GET", f"/api/stacks/{APP}/hdd-data") out["hdd_data"] = {"code": code, "keep_data_only": ((hd or {}).get("data") or {}).get("keep_data_only")} w.say("(c2) hdd-data -> " + json.dumps(out["hdd_data"])) for lang in ("hu", "en"): code, rr = w.ctl("POST", f"/api/stacks/{APP}/remove?lang={lang}", {"remove_hdd_data": True, "remove_backups": False}) out[f"remove_{lang}"] = {"code": code, "error": (rr or {}).get("error")} w.say(f"(c3) [{lang}] Remove with data -> {code} {out[f'remove_{lang}']['error']!r}") code, rr = w.ctl("POST", f"/api/stacks/{APP}/remove?lang=en", {"remove_hdd_data": False, "remove_backups": True}) out["remove_backups_en"] = {"code": code, "error": (rr or {}).get("error")} w.say(f"(c3) [en] Remove with backups -> {code}") st = w.stack(APP) out["still_deployed"] = st.get("deployed") w.say(f"(c3) still deployed: {st.get('deployed')}") out["mirror_back"] = w.guest(f"mv {MIR}.night-aside {MIR} && ls {MIR}") w.say("(c4) mirror moved back: " + out["mirror_back"].replace(chr(10), " ")) # (d) the WHOLE restore: the button the page offers on the Mentések page for the second copy drill(lambda: (open(f"{CAT}/docker-compose.yml", "w").write(HEAD), open(f"{CAT}/.felhom.yml", "w").write(FY)), "back to the head + the real probe") w.sync_rescan() sess = open(f"{w.SC}/sess{__import__('os').getpid()}.txt").read().strip() csrf = open(f"{w.SC}/csrf{__import__('os').getpid()}.txt").read().strip() t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-X", "POST", "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={APP}", f"{w.BASE}/backup/tier2/unit-restore"], timeout=120) w.say("(d) POST /backup/tier2/unit-restore -> " + (r.stdout or "").split("\n")[0] + " " + str([l for l in (r.stdout or "").split("\n") if l.lower().startswith("location")])) last = None for _ in range(600): c2, dd = w.ctl("GET", "/api/backup/restore-status") d2 = (dd.get("data") or {}) if isinstance(dd, dict) else {} cur = (d2.get("running"), d2.get("message")) if cur != last: w.say(f" restore-status {cur}") last = cur if not d2.get("running") and d2.get("last"): break time.sleep(2) out["restore_msg"] = last out["restore_log"] = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'whole restore|WHOLE restore|Tier-2 whole|Restore-from-unit|hold .*CLEARED' | tail -8") w.say("(d) log:\n" + out["restore_log"]) # (e) w.wait_app(SUB, "/status.php", want=("200",), tries=120, delay=5) out["account"] = bool(F.verify(w, SUB, A, w.say)) out["files_after"] = nf.verify_files(w, SUB, A, {names[0]: files[names[0]], names[2]: files[names[2]]}, w.say) out["kept_beside"] = w.guest(f"find /mnt/felhom-drives/scratch_hdd/userdata/nextcloud -name '*.felhom-*' | sed 's|.*/||'") st = w.stack(APP) out["final"] = {k: st.get(k) for k in ("state", "hold_reason", "update_phase")} w.say("(e) final " + json.dumps(out["final"], ensure_ascii=False) + " kept-beside: " + out["kept_beside"]) json.dump(out, open("../A2/10-held-no-copy-keep-data.json", "w"), indent=2, ensure_ascii=False, default=str) open("../A2/10-held-no-copy-keep-data.log", "w").write("\n".join(w.LOG) + "\n")