117 lines
6.0 KiB
Python
117 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""a1.py <cmd> — Part A1's spike on guest 9202 (drill catalog): the window between an update and the next
|
|
backup, measured, then a restore pressed inside that window. EVIDENCE, NOT PRODUCT: it presses only what a
|
|
person presses (deploy, Update, the backups page's restore) and reads the unit off the disk.
|
|
|
|
unit <label> the unit's manifest + compose images + dump headers + mtimes, and the live facts
|
|
restore <label> [tier2] press the restore (own unit „helyi", or the second-drive unit restore) and read back
|
|
seed / read <label> docmost's seed through its own front door (fixtures.py)
|
|
"""
|
|
import json, os, sys, time
|
|
import walk as w, fixtures
|
|
|
|
APP, SUB = "docmost", "c-docs"
|
|
OUT_DIR = f"{w.EV}/" + os.environ.get("A1OUT", "A1")
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
TOK = w.SC + "/seed-tokens.json"
|
|
|
|
|
|
def out(label):
|
|
return open(f"{OUT_DIR}/{label}.txt", "a", buffering=1)
|
|
|
|
|
|
UNIT_SH = r"""
|
|
for U in /mnt/sys_drive/felhom-data/backups/primary/docmost /mnt/felhom-drives/scratch_hdd/backups/secondary/docmost/recovery-unit; do
|
|
echo "== unit $U"
|
|
[ -d "$U" ] || { echo "(absent)"; continue; }
|
|
python3 - "$U" <<'PY'
|
|
import json, sys, os
|
|
u = sys.argv[1]
|
|
try:
|
|
m = json.load(open(u + "/manifest.json"))
|
|
for k in ("controller_version", "created_at", "image_pins", "db_dumps", "volume_dumps", "dumps_at", "offsite_run_id", "data"):
|
|
if k in m:
|
|
print(f" manifest.{k} = {json.dumps(m[k])}")
|
|
except Exception as e:
|
|
print(" manifest unreadable:", e)
|
|
PY
|
|
echo " compose/ images:"; grep -E '^\s+image:' "$U/compose/docker-compose.yml" 2>/dev/null | sed 's/^ */ /'
|
|
echo " compose/ postgres mount:"; grep -E 'docmost_postgres_data:' "$U/compose/docker-compose.yml" 2>/dev/null | sed 's/^ */ /'
|
|
echo " mtimes (UTC):"; find "$U" -maxdepth 2 -type f -printf ' %TY-%Tm-%TdT%TH:%TM:%TSZ %10s %P\n' | sort -k3 | grep -v pre-restore
|
|
for f in "$U"/db-dumps/*.sql; do [ -f "$f" ] && echo " $(basename $f): $(grep -m1 -o 'Dumped from database version [0-9.]*' "$f")"; done
|
|
done
|
|
echo "== live"
|
|
grep -E '^\s+image:' /opt/docker/stacks/docmost/docker-compose.yml 2>/dev/null | sed 's/^ */ compose /'
|
|
docker exec docmost-postgres sh -c 'echo " PGDATA=$PGDATA PG_VERSION=$(cat $PGDATA/PG_VERSION 2>&1)"' 2>&1
|
|
docker ps -a --filter label=com.docker.compose.project=docmost --format ' {{.Names}} {{.Image}} {{.Status}}'
|
|
docker volume ls -q | grep -i docmost | sed 's/^/ volume /'
|
|
V=$(docker volume inspect docmost_docmost_postgres_data --format '{{.Mountpoint}}' 2>/dev/null); [ -n "$V" ] && { echo " pg volume root:"; ls "$V" | head -30 | tr '\n' ' '; echo; }
|
|
"""
|
|
|
|
|
|
def unit(label):
|
|
o = out(label)
|
|
st = w.stack(APP)
|
|
ac = st.get("app_config") or {}
|
|
o.write(f"# {label} — {time.strftime('%Y-%m-%dT%H:%M:%S%z')}; controller {w.guest('cat /etc/felhom-controller-image').strip()}\n")
|
|
o.write(f"pinned={ac.get('pinned_images')}\ninstalled={json.dumps(ac.get('installed_images'))}\n"
|
|
f"conversion_copy={ac.get('conversion_copy')}\nstate={st.get('state')} phase={st.get('update_phase')}\n")
|
|
o.write(w.guest(UNIT_SH, timeout=120))
|
|
code, d = w.ctl("GET", f"/api/backup/snapshots?stack={APP}")
|
|
o.write(f"restore points offered (GET /api/backup/snapshots): {code} {json.dumps(d.get('data') if isinstance(d, dict) else d)[:600]}\n")
|
|
o.close()
|
|
print(open(f"{OUT_DIR}/{label}.txt").read()[-4000:])
|
|
|
|
|
|
def seed(label):
|
|
toks = json.load(open(TOK)) if os.path.exists(TOK) else {}
|
|
toks[APP] = fixtures.FIXTURES[APP].seed(w, SUB, w.say)
|
|
json.dump(toks, open(TOK, "w"), default=str); os.chmod(TOK, 0o600)
|
|
out(label).write(f"seed: {toks[APP] is not None}\n")
|
|
|
|
|
|
def read(label):
|
|
toks = json.load(open(TOK))
|
|
ok = fixtures.FIXTURES[APP].verify(w, SUB, toks[APP], w.say)
|
|
out(label).write(f"seed reads back through the front door: {ok}\n")
|
|
return ok
|
|
|
|
|
|
def restore(label, tier2=False):
|
|
o = out(label)
|
|
since = w.guest("date -u +%Y-%m-%dT%H:%M:%SZ").strip()
|
|
o.write(f"# {label} — restore pressed at {since} ({'second-drive unit restore' if tier2 else 'own unit, snapshot helyi'})\n")
|
|
if tier2:
|
|
# The „Teljes visszaállítás" button: a FORM post, like the own-unit restore (web/server.go).
|
|
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
|
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
|
r = w.sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}",
|
|
"-X", "POST", "--data-urlencode", f"_csrf={csrf}", "--data-urlencode", f"stack_name={APP}",
|
|
f"{w.BASE}/backup/tier2/unit-restore"], timeout=180)
|
|
loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")]
|
|
o.write(f"POST /backup/tier2/unit-restore -> {(r.stdout or '').split(chr(10))[0].strip()} {loc[:1]}\n")
|
|
t0 = time.time()
|
|
while time.time() - t0 < 900:
|
|
c2, s = w.ctl("GET", "/api/backup/restore-status")
|
|
dd = (s.get("data") or {}) if isinstance(s, dict) else {}
|
|
if not dd.get("running") and time.time() - t0 > 5:
|
|
break
|
|
time.sleep(3)
|
|
o.write(f"restore-status at end: {json.dumps(dd)[:600]}\n")
|
|
else:
|
|
r = w.restore(APP, "helyi")
|
|
o.write(json.dumps({k: v for k, v in r.items() if k != 'observables_after'}, default=str)[:1500] + "\n")
|
|
time.sleep(10)
|
|
o.write("controller lines:\n" + w.guest(
|
|
f"docker logs --since {since} felhom-controller 2>&1 | grep -iE 'docmost|restor|replay|recreat|REFUSED' | grep -v DEBUG | cut -c1-420") + "\n")
|
|
o.write("docmost-postgres log (tail):\n" + w.guest("docker logs --tail 40 docmost-postgres 2>&1 | cut -c1-300") + "\n")
|
|
o.write("docmost log (tail):\n" + w.guest("docker logs --tail 25 docmost 2>&1 | cut -c1-300") + "\n")
|
|
o.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cmd, label = sys.argv[1], sys.argv[2]
|
|
w.login()
|
|
{"unit": unit, "seed": seed, "read": read,
|
|
"restore": lambda l: restore(l, tier2=len(sys.argv) > 3 and sys.argv[3] == "tier2")}[cmd](label)
|