From 5b1972b7f964e04d7e79f0ee666736b46d764c92 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 30 Sep 2026 18:16:45 +0200 Subject: [PATCH] harness: five new front-door fixtures, gitea's installer, HTTPS backends, per-file change names (R-462, R-624, R-735) - fixtures (upgrade_fixtures_box.py): calibre-web (Upload form -> OPDS readback + the served EPUB), wger (web login -> weight entry API), crafty-controller (API v2 roles), uptime-kuma (socket.io polling: setup, login, addStatusPage -> public /api/status-page/); gitea posts its own first-run installer form (R-624's fixable case) and keeps the admin CLI for an installed one. calibre-web and wger run the template's own after_install on the bench, which has none. - R-735: the bench's `password:N:special` now has the controller's shape (randomWithSpecial); test seen failing first (length 32, no special), then 45/45. - upgrade_boxport / the memory watch: a backend traefik reaches over https (loadbalancer.server.scheme) is reached over https on the bench too (crafty-controller). - upgrade-test: files-before/after-detail.json and `files_changed_detail` NAME the files behind a files_may_change mark (R-734's method, now in the harness); test ChangedFiles. - test_catalog_gates: the gate count was stale (9, the runner has 10) and red on main; now 10. Evidence: felhom.eu/documentation/audits/more-night-apps-2026-09-30/ Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- scripts/check-volume-persistence.py | 11 + scripts/test_catalog_gates.py | 5 +- scripts/test_check_volume_persistence.py | 16 + scripts/test_upgrade_bench.py | 11 + scripts/upgrade-test.py | 58 ++- scripts/upgrade_boxport.py | 19 +- scripts/upgrade_fixtures_box.py | 466 ++++++++++++++++++++++- 7 files changed, 565 insertions(+), 21 deletions(-) diff --git a/scripts/check-volume-persistence.py b/scripts/check-volume-persistence.py index d6a1b91..2d62598 100644 --- a/scripts/check-volume-persistence.py +++ b/scripts/check-volume-persistence.py @@ -421,9 +421,20 @@ def _gen(spec, ftype): """ if spec: kind, _, n = spec.partition(":") + n, _, form = n.partition(":") n = int(n) if n.isdigit() else 32 if kind == "hex": return secrets.token_hex(n) + if kind == "password" and form == "special": + # R-735: `password:N:special` — deploy.go randomWithSpecial: N long, a letter or digit first, at least + # one lower, upper, digit and one of passwordSpecials. Pinned by + # test_check_volume_persistence.TestEnvBuilding.test_password_special_carries_the_controllers_shape. + alph, specials = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", "-_.!@#%+=" + while True: + p = "".join(secrets.choice(alph + specials) for _ in range(n)) + if (p[0] not in specials and any(c in specials for c in p) and any(c.islower() for c in p) + and any(c.isupper() for c in p) and any(c.isdigit() for c in p)): + return p if kind in ("password", "secret"): alph = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" return "".join(secrets.choice(alph) for _ in range(n)) diff --git a/scripts/test_catalog_gates.py b/scripts/test_catalog_gates.py index 9ccaff8..ed5567e 100644 --- a/scripts/test_catalog_gates.py +++ b/scripts/test_catalog_gates.py @@ -60,8 +60,9 @@ class CatalogGatesFastTest(unittest.TestCase): spec.loader.exec_module(mod) # STALE UNTIL 2026-09-23: this read 5 gates and 3 fast ones while the table had grown to 7 # (copy-i18n, probe-matches-compose) — the test was red and nobody ran it. Now 9 with the test - # record's two halves; the slow pair stays out of --fast. - self.assertEqual(len(mod.GATES), 9) + # record's two halves; the slow pair stays out of --fast. STALE AGAIN until 2026-09-30: 10 since + # probe-measured joined; the test had been red on main (found by the more-night-apps session). + self.assertEqual(len(mod.GATES), 10) self.assertEqual([g[0] for g in mod.GATES if not g[3]], ["image-resolvable", "volume-persistence"]) self.assertIn("test-record", [g[0] for g in mod.GATES if g[3] and not g[4]]) # runs in CI too # the gates that need git history are the ones the CI half cannot run (R-452's shallow gap) diff --git a/scripts/test_check_volume_persistence.py b/scripts/test_check_volume_persistence.py index 4c9465b..f05d6a7 100644 --- a/scripts/test_check_volume_persistence.py +++ b/scripts/test_check_volume_persistence.py @@ -442,6 +442,22 @@ class TestEnvBuilding(unittest.TestCase): import base64 self.assertEqual(len(base64.b64decode(v[len("base64:"):])), 32) + def test_password_special_carries_the_controllers_shape(self): + """R-735: `password:N:special` must be what the controller's randomWithSpecial mints (deploy.go L1335): + N long, a letter or digit first, at least one lower, upper, digit and one of `-_.!@#%+=`. Without the + special character calibre-web's own rule refuses the password and the bench cannot seed the app. + COMPANION RED-PROOF: before the fix `_gen` returned letters and digits only; this test failed.""" + felhom = "deploy_fields:\n - env_var: ADMIN_PASSWORD\n type: password\n generate: \"password:24:special\"\n" + for _ in range(50): + v = cvp.build_env("calibre-web", felhom, "${ADMIN_PASSWORD}")["ADMIN_PASSWORD"] + self.assertEqual(len(v), 24, v) + self.assertTrue(any(c in "-_.!@#%+=" for c in v), f"no special character: {v}") + self.assertNotIn(v[0], "-_.!@#%+=") + for cls in ("abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "0123456789"): + self.assertTrue(any(c in cls for c in v), f"missing one of {cls[:3]}…: {v}") + plain = "deploy_fields:\n - env_var: P\n type: password\n generate: \"password:16\"\n" + self.assertTrue(cvp.build_env("x", plain, "${P}")["P"].isalnum()) # the plain form is unchanged + def test_deploy_fields_block_ends_at_the_next_top_level_key(self): felhom = "deploy_fields:\n - env_var: A\n type: text\napp_info:\n tagline: x\n" self.assertEqual([f["env_var"] for f in cvp.parse_deploy_fields(felhom)], ["A"]) diff --git a/scripts/test_upgrade_bench.py b/scripts/test_upgrade_bench.py index 9702a73..1088e2e 100644 --- a/scripts/test_upgrade_bench.py +++ b/scripts/test_upgrade_bench.py @@ -70,5 +70,16 @@ class ClearScratch(unittest.TestCase): self.assertTrue(os.path.exists(keep)) +class ChangedFiles(unittest.TestCase): + """2026-09-30: a `files_may_change` mark names its files (R-734's method, now in the harness).""" + + def test_names_changed_added_removed_and_nothing_else(self): + before = {"/d": {"a.txt": "1:aa", "b.txt": "1:bb", "gone": "1:cc"}} + after = {"/d": {"a.txt": "1:aa", "b.txt": "2:bd", "new": "1:dd"}} + self.assertEqual(ut.changed_files(before, after), + ["/d/b.txt (changed)", "/d/gone (removed)", "/d/new (added)"]) + self.assertEqual(ut.changed_files(before, before), []) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/upgrade-test.py b/scripts/upgrade-test.py index 8800ef2..d06c30f 100755 --- a/scripts/upgrade-test.py +++ b/scripts/upgrade-test.py @@ -370,12 +370,14 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P paths = LOAD_PATHS.get(app, ["/"]) native = fx.FIXTURES.get(app) cname, port = getattr(native, "container", app), getattr(native, "port", 80) + scheme = "http" if native is None and boxport is not None: # a box-fixture app: load the container traefik routes `/` to, at its own port rts = boxport.routes((workdir / "docker-compose.yml").read_text()) root = [r for r in rts if not r[0]] or rts if root: cname, port = root[0][1], root[0][2] + scheme = root[0][3] if len(root[0]) > 3 else "http" # an HTTPS backend (crafty-controller) target = container_ip(cname) stop = threading.Event() hits = {"n": 0, "codes": {}} @@ -383,7 +385,8 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P class _NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *a, **k): return None # a 30x is the app ANSWERING; following it to the unreachable domain is not load - opener = urllib.request.build_opener(_NoRedirect) + import ssl + opener = urllib.request.build_opener(_NoRedirect, urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) host = None try: envtxt = (workdir / ".env").read_text() @@ -398,7 +401,7 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P def worker(i): k = 0 while not stop.is_set(): - url = f"http://{target}:{port}{paths[(i + k) % len(paths)]}" + url = f"{scheme}://{target}:{port}{paths[(i + k) % len(paths)]}" k += 1 try: req = urllib.request.Request(url, headers={"Host": host, "X-Forwarded-Proto": "https"} if host else {}) @@ -565,6 +568,51 @@ def bind_tree_hash(project: str, workdir: Path) -> dict: return out +def bind_tree_files(project: str, workdir: Path) -> dict: + """{bind source dir: {relpath: "size:sha256[:16]"}} — the same walk as bind_tree_hash, per FILE, so a + `files_may_change` mark can NAME the files it saw change (R-734's method, 2026-09-30: the tree hash alone + could not say which file moved). Files above 64 MiB carry size and mtime only, as in bind_tree_hash.""" + import hashlib + r = compose(workdir, project, "ps", "-aq", timeout=120) + srcs = set() + for cid in (r.stdout or "").split(): + info = cvp._inspect(cid) or {} + for m in info.get("Mounts") or []: + if m.get("Type") == "bind" and os.path.isdir(m.get("Source") or "") \ + and not (m.get("Source") or "").startswith(("/var/run", "/run", "/etc", "/proc", "/sys")): + srcs.add(m["Source"]) + out = {} + for src in sorted(srcs): + files = {} + for dp, _, fn in os.walk(src): + for f in fn: + fp = os.path.join(dp, f) + try: + st = os.lstat(fp) + if st.st_size <= 64 * 1024 * 1024 and os.path.isfile(fp) and not os.path.islink(fp): + with open(fp, "rb") as fh: + d = hashlib.sha256(fh.read()).hexdigest()[:16] + else: + d = "mtime-%d" % int(st.st_mtime) + files[os.path.relpath(fp, src)] = "%d:%s" % (st.st_size, d) + except OSError: + files[os.path.relpath(fp, src)] = "unreadable" + out[src] = files + return out + + +def changed_files(before: dict, after: dict) -> list: + """["/ (added|removed|changed)"] between two bind_tree_files results.""" + out = [] + for src in sorted(set(before) | set(after)): + b, a = before.get(src) or {}, after.get(src) or {} + for rel in sorted(set(b) | set(a)): + if b.get(rel) != a.get(rel): + kind = "added" if rel not in b else "removed" if rel not in a else "changed" + out.append(f"{src}/{rel} ({kind})") + return out + + # --- the PostgreSQL major conversion on the bench (`09` §6.4 part 10, harness v4) --------------------- # # The postgres image converts nothing and refuses an older major's datadir (R-463); 18 refuses even an @@ -793,6 +841,8 @@ def run_edge(edge_id: str) -> dict: files_before = bind_tree_hash(project, workdir) (ev / "files-before.json").write_text(json.dumps(files_before, indent=2)) + detail_before = bind_tree_files(project, workdir) + (ev / "files-before-detail.json").write_text(json.dumps(detail_before, indent=2)) # --- 4. TO --- swap_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") @@ -846,8 +896,12 @@ def run_edge(edge_id: str) -> dict: (ev / "files-after.json").write_text(json.dumps(files_after, indent=2)) rec["files_changed"] = sorted(k for k in set(files_before) | set(files_after) if files_before.get(k) != files_after.get(k)) + detail_after = bind_tree_files(project, workdir) + (ev / "files-after-detail.json").write_text(json.dumps(detail_after, indent=2)) + rec["files_changed_detail"] = changed_files(detail_before, detail_after)[:200] if rec["files_changed"]: rec["marks"] = sorted(set(rec["marks"]) | {"files_may_change"}) + say(f"files_may_change — the files: {rec['files_changed_detail'][:20]}") say(f"files_may_change: the bind-mounted tree changed under {rec['files_changed']}") # --- 5b. the MEMORY WATCH — only for an edge that just read back; a failed one is decided --- diff --git a/scripts/upgrade_boxport.py b/scripts/upgrade_boxport.py index 8abc57f..93c9f58 100644 --- a/scripts/upgrade_boxport.py +++ b/scripts/upgrade_boxport.py @@ -29,13 +29,16 @@ import upgrade_fixtures_box28 as _box28 ROUTE_RULE_RE = re.compile(r"traefik\.http\.routers\.([A-Za-z0-9_-]+)\.rule[=:]\s*[\"']?(.+?)[\"']?\s*$") ROUTE_SVC_RE = re.compile(r"traefik\.http\.routers\.([A-Za-z0-9_-]+)\.service[=:]\s*[\"']?([A-Za-z0-9_-]+)") LB_PORT_RE = re.compile(r"traefik\.http\.services\.([A-Za-z0-9_-]+)\.loadbalancer\.server\.port[=:]\s*[\"']?(\d+)") +# 2026-09-30 (crafty-controller): a backend that speaks HTTPS says so to traefik; the bench must too. +LB_SCHEME_RE = re.compile(r"traefik\.http\.services\.([A-Za-z0-9_-]+)\.loadbalancer\.server\.scheme[=:]\s*[\"']?(https?)") PATH_RE = re.compile(r"PathPrefix\(`([^`]+)`\)") SERVICE_RE = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$") CNAME_RE = re.compile(r"^\s+container_name:\s*[\"']?([^\s\"']+)") def routes(compose_text): - """[(path_prefixes, container, port)] for every compose service traefik routes to.""" + """[(path_prefixes, container, port, scheme)] for every compose service traefik routes to. The scheme is + the 4th element so every reader of [0]..[2] is unchanged.""" out, cur, cname, labels = [], None, {}, {} for line in compose_text.splitlines(): m = SERVICE_RE.match(line) @@ -50,9 +53,9 @@ def routes(compose_text): if "traefik." in line: labels.setdefault(cur, []).append(line.strip().lstrip("- ").strip()) for svc, ls in labels.items(): - rules, rsvc, ports = {}, {}, {} + rules, rsvc, ports, schemes = {}, {}, {}, {} for l in ls: - for rx, d in ((ROUTE_RULE_RE, rules), (ROUTE_SVC_RE, rsvc), (LB_PORT_RE, ports)): + for rx, d in ((ROUTE_RULE_RE, rules), (ROUTE_SVC_RE, rsvc), (LB_PORT_RE, ports), (LB_SCHEME_RE, schemes)): mm = rx.search(l) if mm: d[mm.group(1)] = mm.group(2) @@ -62,7 +65,7 @@ def routes(compose_text): prefixes = [] for r, rule in rules.items(): prefixes += PATH_RE.findall(rule) - out.append((prefixes, cname.get(svc, svc), port)) + out.append((prefixes, cname.get(svc, svc), port, next(iter(schemes.values()), "http"))) return out @@ -83,12 +86,12 @@ class Venue: def _target(self, path): best, blen = None, -1 - for prefixes, container, port in self.routes: + for prefixes, container, port, scheme in self.routes: if not prefixes and blen < 0: - best, blen = (container, port), 0 + best, blen = (container, port, scheme), 0 for p in prefixes: if path.startswith(p) and len(p) > blen: - best, blen = (container, port), len(p) + best, blen = (container, port, scheme), len(p) return best def sh(self, args, timeout=300, inp=None): @@ -115,7 +118,7 @@ class Venue: args += ["-X", method] if data is not None: args += ["--data-binary", "@-"] - args += list(extra) + ["http://%s:%d%s" % (ip, t[1], path)] + args += list(extra) + ["%s://%s:%d%s" % (t[2], ip, t[1], path)] r = self.sh(args, timeout=timeout + 30, inp=data) body, _, code = (r.stdout or "").rpartition("\n") return r.returncode, code.strip(), body diff --git a/scripts/upgrade_fixtures_box.py b/scripts/upgrade_fixtures_box.py index a7e0601..b59803a 100644 --- a/scripts/upgrade_fixtures_box.py +++ b/scripts/upgrade_fixtures_box.py @@ -19,7 +19,7 @@ verify() must ask the APP, never the filesystem: a migration is supposed to rewr Where a fixture can prove itself (a negative control that must read as absent) it does so on EVERY call, so a readback that has broken into always saying "found" fails instead of passing everything. """ -import base64, json, re, secrets, time +import base64, json, os, re, secrets, time def _gx(w, container, *cmd, timeout=240): @@ -172,21 +172,55 @@ class BookStack: # ============================================================================================= class Gitea: - """Gitea's own admin CLI creates the first user; its own REST API (basic auth) then creates a - repository and reads it back. Both are the app's own interfaces.""" + """The first user comes from gitea's own FIRST-RUN INSTALLER (R-624, 2026-09-30): the template sets no + INSTALL_LOCK, so a fresh instance serves the installer form at `/`, and `gitea admin user create` + refuses (`MustInstalled()`). The household fills that form in; so does this fixture — `POST /` with the + form's OWN default values (read from the page, never typed) plus an admin account. Measured on the bench + 2026-09-30 (1.27.0): no CSRF field on the install form; 200, gitea restarts its web server in-process, and + the admin's Basic auth answers `/api/v1/user` 200 within seconds. On a box the setup gate (decision 46) is + in front; walk.app_curl passes it as the household does. An instance that is ALREADY installed falls back + to the admin CLI (the old route). Its own REST API (basic auth) then creates a repository and reads it + back. All of these are the app's own interfaces.""" sub = "git" + FORM_DEFAULTS = ("db_path", "app_name", "repo_root_path", "lfs_root_path", "run_user", "domain", + "ssh_port", "http_port", "app_url", "log_root_path") def seed(self, w, sub, say): if not w.wait_app(sub, "/", want=("200", "302")): return None user = "drill" + secrets.token_hex(3) pw = "Drill-" + secrets.token_hex(10) - out = _gx(w, "gitea", "su", "git", "-c", - f"gitea admin user create --username {user} --password {pw} " - f"--email {user}@gate.invalid --admin --must-change-password=false") - say(f" gitea: admin user create :: {' '.join(out.split())[:140]}") - if "has been successfully created" not in out and "successfully created" not in out: - return None + rc, code, page = w.app_curl(sub, "/") + if 'name="db_type"' in (page or "") and 'name="admin_name"' in (page or ""): + args = ["--data-urlencode", "db_type=sqlite3", "--data-urlencode", "password_algorithm=pbkdf2"] + for n in self.FORM_DEFAULTS: + m = re.search(r'name="%s" value="([^"]*)"' % n, page) + if m: + args += ["--data-urlencode", f"{n}={m.group(1)}"] + args += ["--data-urlencode", f"admin_name={user}", "--data-urlencode", f"admin_passwd={pw}", + "--data-urlencode", f"admin_confirm_passwd={pw}", + "--data-urlencode", f"admin_email={user}@gate.invalid"] + rc, code, body = w.app_curl(sub, "/", *args, method="POST", timeout=120) + say(f" gitea: first-run installer POST / http={code}") + ok = False + for _ in range(40): # the installer restarts gitea's web server in-process + rc, c2, _ = w.app_curl(sub, "/api/v1/user", "-u", f"{user}:{pw}", timeout=15) + if c2 == "200": + ok = True + break + time.sleep(3) + if not ok: + self.tried = f"installer POST / -> {code}; the admin never authenticated (last {c2})" + say(f" gitea: {self.tried} :: {' '.join((body or '').split())[:160]}") + return None + say(" gitea: installed through its own first-run form; the admin authenticates") + else: + out = _gx(w, "gitea", "su", "git", "-c", + f"gitea admin user create --username {user} --password {pw} " + f"--email {user}@gate.invalid --admin --must-change-password=false") + say(f" gitea: already installed — admin user create :: {' '.join(out.split())[:140]}") + if "successfully created" not in out: + return None repo = "drillrepo" + secrets.token_hex(3) rc, code, body = w.app_curl(sub, "/api/v1/user/repos", "-u", f"{user}:{pw}", "-H", "Content-Type: application/json", @@ -1417,7 +1451,421 @@ class Outline: return found +# ============================================================================================= +class CalibreWeb: + """Calibre-Web Automated (2026-09-30, R-462). THE FRONT DOOR is the household's own „Upload" button: log in + through the login form (Flask-WTF CSRF token read from the page), then `POST /upload` with the book, exactly + the form the page posts. The other door — dropping a file into the ingest folder (`${IMPORT_PATH}/calibre`, + reached through FileBrowser) — is NOT used: from here it would be a file planted in a mount (R-156). + Measured on the bench 2026-09-30: uploads are ON in a fresh v4.0.6; the book lands in the household's + library folder (`${USERDATA_PATH}/media/books// (<id>)/`, backup class `mandatory`). + + THE ADMIN PASSWORD. On a box the product's `after_install` (the app's own CLI, `cps.py -s admin:<pw>`) sets + it from the deploy field, and the walk holds the value it generated. The bench runs no `after_install`, so + when the generated password does not log in, the fixture runs THE SAME command the product runs — the + app's own CLI inside its own container — and says so. + + READBACK: the app's OPDS feed (HTTP Basic, the route e-readers use): the search must list the title, and the + book's own download must be the uploaded book (the marker read out of the EPUB the app serves). Negative + controls on every call: a wrong password must answer 401, and a title that cannot exist must not be found.""" + sub = "books" + + @staticmethod + def _epub(marker): + import io, zipfile + buf = io.BytesIO() + z = zipfile.ZipFile(buf, "w") + z.writestr(zipfile.ZipInfo("mimetype"), "application/epub+zip") + z.writestr("META-INF/container.xml", + '<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">' + '<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>' + '</rootfiles></container>') + z.writestr("OEBPS/content.opf", + '<?xml version="1.0" encoding="UTF-8"?><package xmlns="http://www.idpf.org/2007/opf" ' + 'unique-identifier="bid" version="2.0"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/" ' + f'xmlns:opf="http://www.idpf.org/2007/opf"><dc:title>{marker}</dc:title>' + '<dc:creator opf:role="aut">Drill Author</dc:creator><dc:language>en</dc:language>' + f'<dc:identifier id="bid">urn:uuid:{marker}</dc:identifier></metadata><manifest>' + '<item id="c1" href="c1.xhtml" media-type="application/xhtml+xml"/>' + '<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/></manifest>' + '<spine toc="ncx"><itemref idref="c1"/></spine></package>') + z.writestr("OEBPS/toc.ncx", + '<?xml version="1.0"?><ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">' + f'<head><meta name="dtb:uid" content="urn:uuid:{marker}"/></head><docTitle><text>{marker}</text>' + '</docTitle><navMap><navPoint id="n1" playOrder="1"><navLabel><text>One</text></navLabel>' + '<content src="c1.xhtml"/></navPoint></navMap></ncx>') + z.writestr("OEBPS/c1.xhtml", + '<?xml version="1.0" encoding="UTF-8"?><html xmlns="http://www.w3.org/1999/xhtml"><head>' + f'<title>{marker}

{marker}-body the household keeps this.

') + z.close() + return buf.getvalue() + + def _opds(self, w, sub, pw, path): + return w.app_curl(sub, path, "-u", f"admin:{pw}") + + def seed(self, w, sub, say): + import os, tempfile + if not w.wait_app(sub, "/login", want=("200",), tries=72): + return None + pw = (w.GENERATED.get("calibre-web") or {}).get("ADMIN_PASSWORD") or "" + rc, code, _ = self._opds(w, sub, pw, "/opds") + if code != "200": + say(f" calibre-web: the generated admin password does not log in (opds http={code}) — running the " + "template's own after_install command (the app's CLI, as the product does after an install)") + import shlex # as the template's after_install: `user: abc` (docker exec -u keeps the image's PATH; `su` does not) + out = w.guest("docker exec -u abc calibre-web python3 /app/calibre-web-automated/cps.py -p /config/app.db -s " + + shlex.quote(f"admin:{pw}") + " 2>&1") + say(f" calibre-web: after_install :: {' '.join(out.split())[-80:]}") + rc, code, _ = self._opds(w, sub, pw, "/opds") + if code != "200": + self.tried = f"admin login after after_install -> {code}" + return None + jar = tempfile.mktemp(prefix="cw-jar-") + try: + rc, code, page = w.app_curl(sub, "/login", "-c", jar, "-b", jar) + m = re.search(r'name="csrf_token" value="([^"]+)"', page or "") + if not m: + self.tried = f"GET /login -> {code}, no csrf_token" + return None + rc, code, _ = w.app_curl(sub, "/login", "-c", jar, "-b", jar, "--data-urlencode", f"csrf_token={m.group(1)}", + "--data-urlencode", "username=admin", "--data-urlencode", f"password={pw}", + "--data-urlencode", "remember_me=on", method="POST") + rc, code, home = w.app_curl(sub, "/", "-c", jar, "-b", jar) + m = re.search(r'name="csrf_token" value="([^"]+)"', home or "") + has_form = 'action="/upload"' in (home or "") + if code != "200" or not has_form or not m: + self.tried = f"login -> home {code}, upload form present={has_form}" + say(f" calibre-web: {self.tried}") + return None + marker = "upg" + secrets.token_hex(6) + book = tempfile.mktemp(prefix="cw-", suffix=".epub") + open(book, "wb").write(self._epub(marker)) + rc, code, out = w.app_curl(sub, "/upload", "-c", jar, "-b", jar, "-H", f"X-CSRFToken: {m.group(1)}", + "-F", f"csrf_token={m.group(1)}", + "-F", f"btn-upload=@{book};type=application/epub+zip", method="POST") + os.unlink(book) + say(f" calibre-web: POST /upload http={code} {out[:80]}") + if code != "200": + self.tried = f"POST /upload -> {code} {out[:120]}" + return None + finally: + if os.path.exists(jar): + os.unlink(jar) + t = {"pw": pw, "marker": marker} + for _ in range(24): # the upload is a task; the library entry follows within seconds + rc, code, feed = self._opds(w, sub, pw, f"/opds/search/{marker}") + if marker in (feed or ""): + say(f" calibre-web: seeded book {marker} is in the library") + return t + time.sleep(5) + self.tried = "uploaded, but the book never appeared in the OPDS search" + return None + + def verify(self, w, sub, t, say): + import io, os, tempfile, zipfile + if not w.wait_app(sub, "/login", want=("200",), tries=72): + say(" calibre-web: the app never served /login") + return False + rc, code, _ = self._opds(w, sub, "wrong-" + secrets.token_hex(6), "/opds") + if code != "401": + say(f" calibre-web: READBACK UNUSABLE — a wrong password answered {code}") + return False + ghost = "upg" + secrets.token_hex(6) + rc, code, feed = self._opds(w, sub, t["pw"], f"/opds/search/{ghost}") + if code != "200" or "" in (feed or ""): + say(f" calibre-web: READBACK UNUSABLE — a title that cannot exist: http={code}, entries={(feed or '').count('')}") + return False + feed = "" + for _ in range(12): + rc, code, feed = self._opds(w, sub, t["pw"], f"/opds/search/{t['marker']}") + if code == "200" and f"{t['marker']}" in (feed or ""): + break + time.sleep(5) + listed = f"{t['marker']}" in (feed or "") + m = re.search(r'href="(/opds/download/\d+/epub/?)"', feed or "") + served = False + if listed and m: + f = tempfile.mktemp(prefix="cw-dl-") + rc, code, _ = w.app_curl(sub, m.group(1), "-u", f"admin:{t['pw']}", "-o", f) + try: + z = zipfile.ZipFile(io.BytesIO(open(f, "rb").read())) + served = any(f"{t['marker']}-body" in z.read(n).decode("utf-8", "replace") for n in z.namelist()) + except Exception as e: + say(f" calibre-web: the download is not a readable EPUB (http={code}): {e}") + finally: + if os.path.exists(f): + os.unlink(f) + say(f" calibre-web: readback — listed={listed} download_link={bool(m)} book_content_served={served}") + return listed and served + + +# ============================================================================================= +class Wger: + """wger (2026-09-30, R-462). THE FRONT DOOR is the web login form (`/en/user/login`, Django CSRF), then the + app's own REST API with that session: `POST /api/v2/weightentry/` (a weight on a date — household data in its + SQLite), read back with `GET /api/v2/weightentry/?weight=`. Measured on the bench 2026-09-30 (2.6). + NOT the app-API login (`/allauth/app/v1/auth/login`): it answers 500 on a CORRECT password on this template + (no JWT_PRIVATE_KEY — filed, R-737). THE ADMIN PASSWORD: as calibre-web — the box's `after_install` sets it; + the bench runs the template's own command (password as the last argument) when the generated one does not + log in. Negative controls on every readback: no session answers 403, a weight never entered counts 0.""" + sub = "fitness" + SET_PW = ("import os, sys; sys.path.insert(0, '/home/wger/src'); os.chdir('/home/wger/src'); " + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.main'); import django; django.setup(); " + "from django.contrib.auth.models import User; u = User.objects.get(username='admin'); " + "u.set_password(sys.argv[1]); u.save(); print('FELHOM_AFTER_INSTALL_OK')") + + @staticmethod + def _host(w, sub): + return w.host(sub) if hasattr(w, "host") else f"{sub}.{w.DOMAIN}" + + def _login(self, w, sub, pw, jar): + o = f"https://{self._host(w, sub)}" + hdr = ["-H", f"Origin: {o}", "-H", f"Referer: {o}/en/user/login", "-c", jar, "-b", jar] + rc, code, page = w.app_curl(sub, "/en/user/login", *hdr) + m = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page or "") + if not m: + return None, f"GET /en/user/login -> {code}, no csrf" + rc, code, _ = w.app_curl(sub, "/en/user/login", *hdr, "--data-urlencode", f"csrfmiddlewaretoken={m.group(1)}", + "--data-urlencode", "login=admin", "--data-urlencode", f"password={pw}", method="POST") + jt = open(jar).read() if os.path.exists(jar) else "" + if code != "302" or "sessionid" not in jt: + return None, f"POST /en/user/login -> {code}" + csrf = re.search(r"csrftoken\s+(\S+)", jt) + return hdr + ["-H", f"X-CSRFToken: {csrf.group(1) if csrf else ''}"], "ok" + + def seed(self, w, sub, say): + import shlex, tempfile + if not w.wait_app(sub, "/en/user/login", want=("200",), tries=72): + return None + pw = (w.GENERATED.get("wger") or {}).get("ADMIN_PASSWORD") or "" + jar = tempfile.mktemp(prefix="wger-jar-") + try: + hdr, why = self._login(w, sub, pw, jar) + if hdr is None: + say(f" wger: the generated admin password does not log in ({why}) — running the template's own " + "after_install command (the app's CLI, as the product does after an install)") + out = w.guest("docker exec wger python3 -c " + shlex.quote(self.SET_PW) + " " + shlex.quote(pw) + " 2>&1") + say(f" wger: after_install :: {' '.join(out.split())[-60:]}") + hdr, why = self._login(w, sub, pw, jar) + if hdr is None: + self.tried = f"web login after after_install: {why}" + return None + weight = "%d.%02d" % (60 + secrets.randbelow(60), secrets.randbelow(100)) + rc, code, out = w.app_curl(sub, "/api/v2/weightentry/", *hdr, "-H", "Content-Type: application/json", + data=json.dumps({"date": "2026-09-30T10:00:00Z", "weight": weight}), method="POST") + say(f" wger: POST /api/v2/weightentry/ http={code}") + if code != "201": + self.tried = f"POST /api/v2/weightentry/ -> {code} {out[:120]}" + return None + return {"pw": pw, "weight": weight} + finally: + if os.path.exists(jar): + os.unlink(jar) + + def verify(self, w, sub, t, say): + import tempfile + if not w.wait_app(sub, "/en/user/login", want=("200",), tries=72): + say(" wger: the app never served /en/user/login") + return False + rc, code, _ = w.app_curl(sub, f"/api/v2/weightentry/?weight={t['weight']}") + if code not in ("401", "403"): + say(f" wger: READBACK UNUSABLE — no session answered {code}") + return False + jar = tempfile.mktemp(prefix="wger-jar-") + try: + hdr, why = self._login(w, sub, t["pw"], jar) + if hdr is None: + say(f" wger: login failed: {why}") + return False + rc, code, out = w.app_curl(sub, "/api/v2/weightentry/?weight=199.99", *hdr) + if code != "200" or '"count":0' not in (out or "").replace(" ", ""): + say(f" wger: READBACK UNUSABLE — a weight never entered: http={code} {out[:80]}") + return False + rc, code, out = w.app_curl(sub, f"/api/v2/weightentry/?weight={t['weight']}", *hdr) + ok = code == "200" and f'"weight":"{t["weight"]}"' in (out or "").replace(" ", "") + say(f" wger: readback of the seeded weight entry http={code} found={ok}") + return ok + finally: + if os.path.exists(jar): + os.unlink(jar) + + +# ============================================================================================= +class Crafty: + """Crafty Controller 4 (2026-09-30, R-462). Its own REST API v2, behind its own HTTPS port (the template tells + traefik `scheme=https`; the bench adapter reads that label): `POST /api/v2/auth/login` as `admin` with the + deploy's CRAFTY_PASSWORD (the compose entrypoint writes it into default.json), then `POST /api/v2/roles` — a + role is a record in crafty's own database, created the way its panel creates one — read back with + `GET /api/v2/roles`. Measured on the bench 2026-09-30 (4.10.7): the POST needs `mfa_required` (500 without). + Negative control on every readback: a wrong token must answer 401/403.""" + sub = "minecraft" + + def _token(self, w, sub, pw): + rc, code, out = w.app_curl(sub, "/api/v2/auth/login", "-H", "Content-Type: application/json", + data=json.dumps({"username": "admin", "password": pw}), method="POST") + try: + return json.loads(out)["data"]["token"], code + except Exception: + return None, f"{code} {(out or '')[:120]}" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=90): + return None + pw = (w.GENERATED.get("crafty-controller") or {}).get("CRAFTY_PASSWORD") or "" + tok, why = None, None + for _ in range(12): # crafty answers `/` before its API accepts the seeded admin + tok, why = self._token(w, sub, pw) + if tok: + break + time.sleep(5) + if not tok: + self.tried = f"POST /api/v2/auth/login -> {why}" + return None + role = "upg" + secrets.token_hex(5) + rc, code, out = w.app_curl(sub, "/api/v2/roles", "-H", f"Authorization: Bearer {tok}", + "-H", "Content-Type: application/json", + data=json.dumps({"name": role, "servers": [], "mfa_required": False}), method="POST") + say(f" crafty: POST /api/v2/roles http={code} {out[:60]}") + if code != "200" or '"ok"' not in (out or ""): + self.tried = f"POST /api/v2/roles -> {code} {out[:120]}" + return None + return {"pw": pw, "role": role} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=90): + say(" crafty: the app never answered /") + return False + tok = None + for _ in range(12): + tok, why = self._token(w, sub, t["pw"]) + if tok: + break + time.sleep(5) + if not tok: + say(f" crafty: login failed: {why}") + return False + rc, code, _ = w.app_curl(sub, "/api/v2/roles", "-H", f"Authorization: Bearer x{tok}") + if code not in ("401", "403"): + say(f" crafty: READBACK UNUSABLE — a wrong token answered {code}") + return False + rc, code, out = w.app_curl(sub, "/api/v2/roles", "-H", f"Authorization: Bearer {tok}") + names = [r.get("role_name") for r in (json.loads(out).get("data") or [])] if code == "200" else [] + ok = t["role"] in names + say(f" crafty: readback of the seeded role http={code} found={ok} (roles listed: {len(names)})") + return ok + + +# ============================================================================================= +class UptimeKuma: + """Uptime Kuma 2 (2026-09-30, R-462). Its web page talks to the server over socket.io, and its first-run setup, + login and every edit exist ONLY there — so the fixture speaks socket.io's plain HTTP long-polling transport + (Engine.IO v4: `GET/POST /socket.io/?EIO=4&transport=polling`) through the app's own front door, exactly the + messages the page sends: `setup` (the first admin), `login`, `addStatusPage` (a status page is a record in its + own SQLite). READBACK through its public REST route `GET /api/status-page/` (the page households share), + which must carry the seeded title; negative control: a slug never made must not answer 200 with a title.""" + sub = "status" + RS = "\x1e" + + class _IO: + def __init__(self, w, sub): + self.w, self.sub, self.sid, self.buf, self.ack = w, sub, None, [], 0 + + def _get(self): + rc, code, out = self.w.app_curl(self.sub, f"/socket.io/?EIO=4&transport=polling&sid={self.sid}", timeout=30) + if code != "200": + raise RuntimeError(f"poll http={code} {out[:80]}") + for p in (out or "").split(UptimeKuma.RS): + if p == "2": # ping → pong + self._post("3") + elif p: + self.buf.append(p) + + def _post(self, body): + rc, code, out = self.w.app_curl(self.sub, f"/socket.io/?EIO=4&transport=polling&sid={self.sid}", + "-H", "Content-Type: text/plain;charset=UTF-8", data=body, method="POST") + if code != "200": + raise RuntimeError(f"post http={code} {out[:80]}") + + def open(self): + rc, code, out = self.w.app_curl(self.sub, "/socket.io/?EIO=4&transport=polling") + m = re.search(r'"sid":"([^"]+)"', out or "") + if code != "200" or not m: + raise RuntimeError(f"handshake http={code} {out[:80]}") + self.sid = m.group(1) + self._post("40") + for _ in range(10): + self._get() + if any(p.startswith("40") for p in self.buf): + return self + raise RuntimeError("no socket.io connect") + + def call(self, event, *args): + n = self.ack + self.ack += 1 + self._post(f"42{n}" + json.dumps([event, *args])) + for _ in range(30): + for p in list(self.buf): + if p.startswith(f"43{n}["): + self.buf.remove(p) + return json.loads(p[len(f"43{n}"):])[0] + self._get() + raise RuntimeError(f"no answer to {event}") + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=90): + return None + user, pw = "drill" + secrets.token_hex(3), "Drill-" + secrets.token_hex(10) + "A1" + try: + io = self._IO(w, sub).open() + r = io.call("setup", user, pw) + say(f" uptime-kuma: setup ok={r.get('ok')} {str(r.get('msg'))[:60]}") + if not r.get("ok"): + self.tried = f"socket.io setup -> {r}" + return None + io = self._IO(w, sub).open() + r = io.call("login", {"username": user, "password": pw, "token": ""}) + if not r.get("ok"): + self.tried = f"socket.io login -> {str(r)[:120]}" + return None + title, slug = "Drill " + secrets.token_hex(4), "upg" + secrets.token_hex(4) + r = io.call("addStatusPage", title, slug) + say(f" uptime-kuma: addStatusPage ok={r.get('ok')} {str(r.get('msg'))[:60]}") + if not r.get("ok"): + self.tried = f"socket.io addStatusPage -> {str(r)[:120]}" + return None + except Exception as e: + self.tried = f"socket.io: {e}" + say(f" uptime-kuma: {self.tried}") + return None + return {"user": user, "pw": pw, "title": title, "slug": slug} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=90): + say(" uptime-kuma: the app never answered /") + return False + rc, code, out = w.app_curl(sub, f"/api/status-page/upgnever{secrets.token_hex(4)}") + if code == "200" and '"title"' in (out or ""): + say(f" uptime-kuma: READBACK UNUSABLE — a slug never made answered {code} with a title") + return False + found = False + for _ in range(12): + rc, code, out = w.app_curl(sub, f"/api/status-page/{t['slug']}") + try: + found = code == "200" and json.loads(out)["config"]["title"] == t["title"] + except Exception: + found = False + if found: + break + time.sleep(5) + say(f" uptime-kuma: readback of the seeded status page http={code} found={found}") + return found + + FIXTURES = { + "uptime-kuma": UptimeKuma(), + "crafty-controller": Crafty(), + "wger": Wger(), + "calibre-web": CalibreWeb(), "sparkyfitness": Sparkyfitness(), "rallly": Rallly(), "outline": Outline(),