diff --git a/CHANGELOG.md b/CHANGELOG.md index 4619e63..471393b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +## Every step keeps its own definition — the box can climb one step at a time (2026-09-24, `09` §6.4 part 5, R-653, R-656, R-663) + +**No `image:` line moved.** The box half ships in controller v0.268.0. + +- **`steps/.yml`** — every `update_ladder:` entry but the newest now carries its own complete + compose file. `ladder.step_key` is sha256 of `to` as canonical JSON, first 16 hex; the controller computes + the same string (`stacks.StepKey`, pinned by one shared value). +- **Gate:** `check-test-record.py` rule 4 — an intermediate step with no file, or a file whose own `image:` + lines are not that step's `to`, is refused. Decoys: no file; the right NAME naming the head's image; the + definition under another name (red-proof: rule removed → all three pass wrongly). +- **Writer:** `upgrade-test.py --write-ladder` keeps the superseded head's compose — fixes included — as its + step file when it appends the next entry (red-proof in `test_ladder_writer.py`). +- **Backfill** (`scripts/steps_backfill.py`, one-off): 8 step files for the 7 apps with more than one entry + (emby, ghost, immich, n8n, navidrome, nextcloud, romm ×2), each from the NEWEST commit whose compose names + that step's images — romm's first step comes from `f4eb94f` (the working 768M / two-worker template), not + from `15f9ebf` (the move that OOM-looped). +- **R-653:** the memory watch's load is `reached` only when at least half its requests got an HTTP answer; + otherwise the edge is `inconclusive`, never `proven`. +- **R-656:** the bench clears the app's own scratch drive folders before FROM and says so (never a bare root, + never outside the scratch roots). +- **R-663:** `test_gate_decoys.py` and `test_ladder_writer.py` had been red since the night of 2026-09-23 + (kimai-db and navidrome moved under literals); they now READ the live pins. Suite: 84 cases OK. + ## wishlist fits its first boot; uptime-kuma no longer parks on its database wizard (2026-09-23 night, R-612, R-613) **No `image:` line moved.** Two template fixes, each red-proofed on scratch guest 9202 through the product. diff --git a/scripts/check-test-record.py b/scripts/check-test-record.py index 9c700be..e1d8103 100644 --- a/scripts/check-test-record.py +++ b/scripts/check-test-record.py @@ -19,6 +19,9 @@ WHAT IT CHECKS, per template that carries `update_ladder:` (format and field rul 3. the NEWEST entry's `to` is EXACTLY the compose's current image per service. This is the fact that makes the rule hold without history: a compose moved without a new entry no longer matches its ladder's head, whoever pushed it and however. + 4. every entry but the newest carries its OWN definition at `steps/.yml`, whose + images per service are EXACTLY that entry's `to` (`09` §6.4 part 5: the box pins that file, one + step per press). The name alone is not the fact: the file's own `image:` lines are read. A template WITHOUT a ladder passes here — it has never been moved since the gate existed, and its first move is refused by the twin unless that move brings the first entry. @@ -50,6 +53,18 @@ def check_app(app_dir): for i in range(1, len(entries)): if entries[i].get("from") != entries[i - 1].get("to"): problems.append("entry %d's `from` is not entry %d's `to` — the ladder has a gap" % (i + 1, i)) + for i, e in enumerate(entries[:-1]): + to = e.get("to") + if not isinstance(to, dict) or not to: + continue # already convicted by check_entry + sp = os.path.join(app_dir, ladder.step_file(to)) + if not os.path.isfile(sp): + problems.append("entry %d of %d has no definition at %s — the box climbs one step at a time " + "and needs this step's own compose file" % (i + 1, len(entries), ladder.step_file(to))) + continue + got = ladder.images_in(open(sp, encoding="utf-8").read()) + if got != to: + problems.append("%s names %s, but entry %d's `to` is %s" % (ladder.step_file(to), got, i + 1, to)) if entries: current = ladder.images_in(open(comp, encoding="utf-8").read()) head = entries[-1].get("to") diff --git a/scripts/ladder.py b/scripts/ladder.py index 35415c1..75a26c6 100644 --- a/scripts/ladder.py +++ b/scripts/ladder.py @@ -30,8 +30,16 @@ AN ENTRY (all keys required unless marked): backfilled (optional) "YYYY-MM-DD" — written by the backfill from an EXISTING record, never by a new test; a new move may not carry it +STEP DEFINITIONS (`09` §6.4 part 5, controller v0.268.0): every entry but the NEWEST carries its own +complete compose file at `templates//steps/.yml` — the box climbs one step at a time +and pins exactly that file; the newest step's definition is the template's `docker-compose.yml`. The box's +catalog clone is `--depth 1`, so git history is not a place a box can read a step from, and the commit +that moved an image is not always the definition that works (romm 15f9ebf). `step_key` is computed the +SAME way by the controller (`stacks.StepKey`, pinned by TestLadder_StepKeyMatchesTheCatalog). + Every path that reads or writes the format is here, so the gate and the writer cannot disagree. """ +import hashlib import json import re @@ -191,3 +199,14 @@ def append_entry(felhom_text, e): last = j lines.insert(last + 1, line) return "\n".join(lines) + "\n" + + +def step_key(to): + """The 16-hex name of a step's definition: sha256 of `to` as canonical JSON (keys sorted, no spaces). + The controller computes the same string (stacks.StepKey).""" + return hashlib.sha256(json.dumps(to, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:16] + + +def step_file(to): + """The step definition's path RELATIVE to the template directory.""" + return "steps/%s.yml" % step_key(to) diff --git a/scripts/steps_backfill.py b/scripts/steps_backfill.py new file mode 100644 index 0000000..569c5e4 --- /dev/null +++ b/scripts/steps_backfill.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""steps_backfill.py — ONE-OFF (2026-09-24, `09` §6.4 part 5): write `steps/.yml` for every +ladder entry that is not the newest, from git history. + +WHICH COMMIT, and why not "the commit of the step": the definition a box must pin for a step is the one +the catalog SERVED while that step was the head — the NEWEST commit whose compose images equal the +step's `to`, i.e. the step's images WITH every fix that flowed after they moved. The commit that MOVED +the image can be the broken one: romm's 15f9ebf (5.0.0 -> 5.3.0, 512M, four workers) OOM-looped on +demo-hp; the working definition is the same images under f4eb94f's template (768M, two workers). + +Prints one line per step: app, entry, step key, the commit it came from. Never overwrites a step file. +""" +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import ladder # noqa: E402 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def git(*a): + return subprocess.run(["git", "-C", ROOT] + list(a), capture_output=True, text=True, check=True).stdout + + +def main(): + wrote = 0 + for app in sorted(os.listdir(os.path.join(ROOT, "templates"))): + d = os.path.join(ROOT, "templates", app) + fy = os.path.join(d, ".felhom.yml") + if not os.path.isfile(fy): + continue + entries, _, errs = ladder.parse(open(fy, encoding="utf-8").read()) + if errs or len(entries) < 2: + continue + rel = "templates/%s/docker-compose.yml" % app + commits = git("log", "--format=%H", "--", rel).split() # newest first + for i, e in enumerate(entries[:-1]): + dst = os.path.join(d, ladder.step_file(e["to"])) + if os.path.exists(dst): + print("SKIP %-10s entry %d: %s exists" % (app, i + 1, ladder.step_file(e["to"]))) + continue + found = None + for c in commits: + body = git("show", "%s:%s" % (c, rel)) + if ladder.images_in(body) == e["to"]: + found = (c, body) + break # newest first: the last definition served for this step + if not found: + print("MISS %-10s entry %d: no commit's compose names %s" % (app, i + 1, e["to"])) + continue + os.makedirs(os.path.dirname(dst), exist_ok=True) + open(dst, "w", encoding="utf-8").write(found[1]) + print("WROTE %-10s entry %d/%d -> %s from %s (%s)" % (app, i + 1, len(entries), ladder.step_file(e["to"]), + found[0][:12], git("log", "-1", "--format=%s", found[0]).strip()[:70])) + wrote += 1 + print("steps_backfill: %d step definition(s) written" % wrote) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_gate_decoys.py b/scripts/test_gate_decoys.py index b1457db..0dcef82 100644 --- a/scripts/test_gate_decoys.py +++ b/scripts/test_gate_decoys.py @@ -94,6 +94,7 @@ def commit(clone, msg): def reset(clone): sh(["git", "reset", "-q", "--hard", "HEAD"], cwd=clone) + sh(["git", "clean", "-fdq"], cwd=clone) # a case may ADD a file (a steps/ definition) def case(name, clone, edits, expect_rc, must_contain=(), gate="check-engine-major.py"): @@ -283,6 +284,25 @@ def swap_image(service, frm, to): +def cur_image(clone, relpath, service): + """The service's current image in the clone — read, never typed (R-663).""" + sys.path.insert(0, os.path.join(ROOT, "scripts")) + import ladder as _l + return _l.images_in(io.open(os.path.join(clone, relpath), encoding="utf-8").read())[service] + + +def strip_ladder(t): + """The template WITHOUT its update_ladder block (and the header comment the writer puts above it). + The writer appends the block at the END of the file (ladder.append_entry), so everything from its + first line on goes — a case then builds exactly the ladder it describes, whatever the live catalog + has recorded since (R-663).""" + lines = t.splitlines() + for i, l in enumerate(lines): + if l.startswith("# update_ladder") or l.startswith("update_ladder:"): + return "\n".join(lines[:i]).rstrip("\n") + "\n" + return t + + # ── test record (09 §3 decision 13) ──────────────────────────────────────────────────────────── TR_D1 = "sha256:" + "a" * 64 TR_D2 = "sha256:" + "b" * 64 @@ -299,8 +319,12 @@ def tr_entry(frm, to, digest, verdict="proven", peak=41.0, tight=False, **extra) def tr_append(line, header=True): + """Append one entry line. A template that already HAS a ladder (the writer puts it at the end of + the file) gets the line appended to it; one without gets the block (R-663: navidrome gained a + ladder on 2026-09-23 night, and a second `update_ladder:` key is its own conviction).""" def _fn(t): - block = ("\nupdate_ladder:\n" if header else "") + line + "\n" + has = any(l.startswith("update_ladder:") for l in t.splitlines()) + block = ("\nupdate_ladder:\n" if header and not has else "") + line + "\n" return t.rstrip("\n") + "\n" + block return _fn @@ -349,7 +373,10 @@ def case_tr_static(name, clone, edits, expect_rc, must_contain=(), apps=("navidr def test_record_cases(clone): NC, NF = "templates/navidrome/docker-compose.yml", "templates/navidrome/.felhom.yml" - old, new = "deluan/navidrome:0.64.0", "deluan/navidrome:0.64.1" + # READ, never typed (R-663): the live pin moves with every proven step. + old = cur_image(clone, NC, "navidrome") + new = "deluan/navidrome:0.99.1" + prev = "deluan/navidrome:0.1.0" # an invented older step, for the static cases frm, to = {"navidrome": old}, {"navidrome": new} move = (NC, swap_image("navidrome", old, new)) good = tr_entry(frm, to, {"navidrome": TR_D1}) @@ -367,7 +394,7 @@ def test_record_cases(clone): case_tr_move("INCONCLUSIVE: the registry cannot be asked", clone, [move, (NF, tr_append(good))], 2, ("could not be asked",), {}) case_tr_move("DECOY: the entry only in a COMMENT under update_ladder", clone, - [move, (NF, lambda t: t.rstrip("\n") + "\nupdate_ladder:\n # " + good.strip() + "\n")], 1, + [move, (NF, lambda t: strip_ladder(t).rstrip("\n") + "\nupdate_ladder:\n # " + good.strip() + "\n")], 1, ("holds no entry",), table) case_tr_move("DECOY: the entry only in README.md", clone, [move, ("README.md", lambda t: t + "\n" + good + "\n")], 1, @@ -385,21 +412,42 @@ def test_record_cases(clone): case_tr_move("DECOY: a ref moves in a compose COMMENT only", clone, [(NC, lambda t: t + "\n# was: deluan/navidrome:0.63.2\n")], 0, (), table) print("-- test-record (static): the newest step IS the compose") - case_tr_static("GENUINE: a ladder whose head is the compose", clone, - [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))], 0) + def ladder_of(*lines): + """Replace the template's ladder with exactly these entry lines.""" + return lambda t: tr_append("\n".join(lines))(strip_ladder(t)) + head = tr_entry({"navidrome": prev}, frm, {"navidrome": TR_D1}) + case_tr_static("GENUINE: a ladder whose head is the compose", clone, [(NF, ladder_of(head))], 0) case_tr_static("FACT: the compose moved past the ladder's head", clone, - [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}))), - move], 1, ("not the ladder's newest step",)) + [(NF, ladder_of(head)), move], 1, ("not the ladder's newest step",)) case_tr_static("FACT: a line that is not one JSON entry", clone, - [(NF, lambda t: t + "\nupdate_ladder:\n - from: x\n")], 1, ("not a one-line JSON entry",)) + [(NF, lambda t: strip_ladder(t) + "\nupdate_ladder:\n - from: x\n")], 1, ("not a one-line JSON entry",)) case_tr_static("FACT: a gap between steps", clone, - [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.62.0"}, {"navidrome": "deluan/navidrome:0.63.0"}, {"navidrome": TR_D1}) - + "\n" + tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1})))], + [(NF, ladder_of(tr_entry({"navidrome": "deluan/navidrome:0.0.1"}, {"navidrome": "deluan/navidrome:0.0.2"}, {"navidrome": TR_D1}), head))], 1, ("the ladder has a gap",)) case_tr_static("FACT: a failed verdict sits in the ladder", clone, - [(NF, tr_append(tr_entry({"navidrome": "deluan/navidrome:0.63.2"}, frm, {"navidrome": TR_D1}, verdict="failed")))], + [(NF, ladder_of(tr_entry({"navidrome": prev}, frm, {"navidrome": TR_D1}, verdict="failed")))], 1, ("not allowed in a ladder",)) + # ── `09` §6.4 part 5 (v0.268.0 on the box): every step but the newest carries its OWN definition + # at steps/.yml — the box climbs one step at a time and pins that file. ────────── + print("-- test-record (static): every intermediate step carries its own definition") + sys.path.insert(0, os.path.join(ROOT, "scripts")) + import ladder as _l + mid = {"navidrome": "deluan/navidrome:0.2.0"} + two = ladder_of(tr_entry({"navidrome": prev}, mid, {"navidrome": TR_D1}), + tr_entry(mid, frm, {"navidrome": TR_D1})) + step_rel = "templates/navidrome/" + _l.step_file(mid) + step_body = lambda t: swap_image("navidrome", old, mid["navidrome"])(io.open(os.path.join(clone, NC), encoding="utf-8").read()) + case_tr_static("FACT: a two-step ladder with NO steps/ file for the first step", clone, + [(NF, two)], 1, ("has no definition",)) + case_tr_static("GENUINE: a two-step ladder whose first step carries its definition", clone, + [(NF, two), (step_rel, step_body)], 0) + case_tr_static("DECOY: the steps/ file has the right NAME and names the head's image", clone, + [(NF, two), (step_rel, lambda t: io.open(os.path.join(clone, NC), encoding="utf-8").read())], + 1, ("names",)) + case_tr_static("DECOY: the step's definition sits beside the template under another name", clone, + [(NF, two), ("templates/navidrome/steps/0.2.0.yml", step_body)], 1, ("has no definition",)) + def main(): gate = os.path.join(ROOT, "scripts", "check-engine-major.py") if not os.path.isfile(gate): @@ -409,6 +457,12 @@ def main(): try: KIMAI = "templates/kimai/docker-compose.yml" DOCMOST = "templates/docmost/docker-compose.yml" + # The engine ref is READ from the clone, not typed: the night of 2026-09-23 moved kimai-db + # 11.6 -> 11.8 through its own test record, and a literal here broke every case below + # ("fixture drifted") without a single gate changing. R-663. + KDB = cur_image(clone, KIMAI, "kimai-db") + if not KDB.startswith("mariadb:11."): + raise SystemExit("kimai-db is %s — the cases below assume a MariaDB 11 line; fixture drifted" % KDB) # ── THE FACTS: these must be refused ───────────────────────────────────────────────── out = case("FACT: docmost-postgres 16-alpine -> 17-alpine bundled with the app bump", clone, @@ -424,20 +478,20 @@ def main(): # bookstack 0b73e5e shape — two migrations behind one edge — and stays refused. case("FACT: kimai-db 11.6 -> 12.3 BUNDLED with the kimai app bump", clone, [(KIMAI, lambda t: swap_image("kimai", "kimai/kimai2:apache-2.57.0", "kimai/kimai2:apache-2.58.0")( - swap_image("kimai-db", "mariadb:11.6", "mariadb:12.3")(t)))], + swap_image("kimai-db", KDB, "mariadb:12.3")(t)))], expect_rc=1, must_contain=("IN THE SAME COMMIT as kimai", "OWN EDGE", "R-450")) case("FACT: kimai-db mariadb:11.6 -> mariadb:lts (major unreadable)", clone, - [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:lts"))], + [(KIMAI, swap_image("kimai-db", KDB, "mariadb:lts"))], expect_rc=2, must_contain=("INCONCLUSIVE",)) # ── THE GENUINE ARTICLES: these must pass ──────────────────────────────────────────── - case("GENUINE: kimai-db mariadb:11.6 -> 11.8 (within major)", clone, - [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:11.8"))], + case("GENUINE: kimai-db mariadb:11.x -> 11.99 (within major)", clone, + [(KIMAI, swap_image("kimai-db", KDB, "mariadb:11.99"))], expect_rc=0, must_contain=("engine-major gate OK",)) # R-469: the LIFT itself. A MariaDB major ALONE in its template is now permitted, and the # gate says so by name rather than passing in silence. case("GENUINE: kimai-db mariadb:11.6 -> 12.3 ALONE (the R-469 lift)", clone, - [(KIMAI, swap_image("kimai-db", "mariadb:11.6", "mariadb:12.3"))], + [(KIMAI, swap_image("kimai-db", KDB, "mariadb:12.3"))], expect_rc=0, must_contain=("ALLOWED", "kimai-db", "mariadb 11 -> 12", "R-469")) # ── THE DECOYS: the label moves, the fact does not — these must pass ───────────────── diff --git a/scripts/test_ladder_writer.py b/scripts/test_ladder_writer.py index 89d65dd..2fe9460 100644 --- a/scripts/test_ladder_writer.py +++ b/scripts/test_ladder_writer.py @@ -28,7 +28,14 @@ spec.loader.exec_module(ut) D = "sha256:" + "c" * 64 -def bench(verdict="proven", peak=0.41, marks=(), frm="0.64.0", to="0.64.1"): +# READ, never typed (R-663): navidrome moved 0.64.0 -> 0.64.1 on 2026-09-23 night and a literal here +# broke two of these tests without the writer changing. +LIVE = ladder.images_in(open(os.path.join(ROOT, "templates", "navidrome", "docker-compose.yml")).read())["navidrome"].split(":")[1] +NEXT = LIVE + "-next" +AFTER = LIVE + "-after" + + +def bench(verdict="proven", peak=0.41, marks=(), frm=LIVE, to=NEXT): return {"harness_version": 3, "app": "navidrome", "verdict": verdict, "from": {"navidrome": "deluan/navidrome:" + frm}, "to": {"navidrome": "deluan/navidrome:" + to}, "measured_at": "2026-09-23T22:00:00Z", "marks": list(marks), @@ -41,11 +48,12 @@ class WriterTest(unittest.TestCase): shutil.copytree(os.path.join(ROOT, "templates", "navidrome"), os.path.join(self.tmp, "templates", "navidrome")) self.fy = os.path.join(self.tmp, "templates", "navidrome", ".felhom.yml") - # start from a template WITHOUT a ladder, at 0.64.0 (the live pin tonight) + # start from a template WITHOUT a ladder, at the live pin text = open(self.fy).read() if "update_ladder:" in text: text = text[:text.index("\n# update_ladder")] + "\n" open(self.fy, "w").write(text) + shutil.rmtree(os.path.join(self.tmp, "templates", "navidrome", "steps"), ignore_errors=True) self._orig = image_digest.resolve image_digest.resolve = lambda ref: (D, None) @@ -67,11 +75,11 @@ class WriterTest(unittest.TestCase): self.assertEqual(rc, 0, out) entries, _, errs = ladder.parse(open(self.fy).read()) self.assertEqual(errs, []) - self.assertEqual(entries[-1]["to"], {"navidrome": "deluan/navidrome:0.64.1"}) + self.assertEqual(entries[-1]["to"], {"navidrome": "deluan/navidrome:" + NEXT}) self.assertEqual(entries[-1]["memory_peak_pct"], 41.0) # a PERCENT, from the watch's fraction self.assertEqual(entries[-1]["digest"], {"navidrome": D}) comp = open(os.path.join(self.tmp, "templates", "navidrome", "docker-compose.yml")).read() - self.assertEqual(ladder.images_in(comp), {"navidrome": "deluan/navidrome:0.64.1"}) + self.assertEqual(ladder.images_in(comp), {"navidrome": "deluan/navidrome:" + NEXT}) import subprocess r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp, "navidrome"], capture_output=True, text=True) @@ -88,7 +96,7 @@ class WriterTest(unittest.TestCase): self.assertNotIn("update_ladder:", open(self.fy).read()) def test_refuses_when_the_template_is_not_at_from(self): - rc, out = self.run_writer(bench(frm="0.63.2")) + rc, out = self.run_writer(bench(frm="0.0.1")) self.assertEqual(rc, 1) self.assertIn("the template is at", out) @@ -98,6 +106,27 @@ class WriterTest(unittest.TestCase): e = ladder.parse(open(self.fy).read())[0][-1] self.assertEqual(e["marks"], {"files_may_change": True, "needs_person": None, "memory_tight": True}) + def test_a_second_step_keeps_the_first_steps_definition(self): + """`09` §6.4 part 5: writing step 2 turns step 1 into an intermediate step — its OWN definition + (the compose as it stood, fixes included) is kept at steps/.yml, and the gate accepts the + two-step ladder. RED-PROOF (REPORT.md): drop the STEP block in write_ladder — the gate refuses + at "has no definition".""" + rc, out = self.run_writer(bench()) + self.assertEqual(rc, 0, out) + comp_p = os.path.join(self.tmp, "templates", "navidrome", "docker-compose.yml") + # a fix flows into the step-1 definition after it was published (the romm f4eb94f shape) + open(comp_p, "a").write("# a fix that flowed after step 1\n") + at_step1 = open(comp_p).read() + rc, out = self.run_writer(bench(frm=NEXT, to=AFTER)) + self.assertEqual(rc, 0, out) + sp = os.path.join(self.tmp, "templates", "navidrome", ladder.step_file({"navidrome": "deluan/navidrome:" + NEXT})) + self.assertTrue(os.path.isfile(sp), out) + self.assertEqual(open(sp).read(), at_step1, "the step file must be the definition AS SERVED, fixes included") + import subprocess + r = subprocess.run([sys.executable, os.path.join(HERE, "check-test-record.py"), "--root", self.tmp, + "navidrome"], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/test_upgrade_bench.py b/scripts/test_upgrade_bench.py new file mode 100644 index 0000000..9702a73 --- /dev/null +++ b/scripts/test_upgrade_bench.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""test_upgrade_bench.py — the test bench's two night-of-2026-09-23 faults (R-653, R-656). No Docker. + + python3 scripts/test_upgrade_bench.py +""" +import importlib.util +import os +import shutil +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location("upgrade_test_mod", os.path.join(HERE, "upgrade-test.py")) +ut = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ut) + + +class LoadVerdict(unittest.TestCase): + """R-653. RED-PROOF (REPORT.md): make load_verdict always return "reached" — the ghost case fails.""" + + def test_every_request_errored_is_inconclusive(self): + # ghost's first watch, 2026-09-23 night: 11 797 requests, all `err` + self.assertEqual(ut.load_verdict(11797, {"err": 11797}), "inconclusive") + + def test_answers_of_any_code_are_the_app_answering(self): + self.assertEqual(ut.load_verdict(11429, {"200": 5712, "401": 5717}), "reached") + self.assertEqual(ut.load_verdict(10, {"302": 5, "err": 5}), "reached") + + def test_under_half_is_inconclusive_and_none_is_inconclusive(self): + self.assertEqual(ut.load_verdict(10, {"200": 4, "err": 6}), "inconclusive") + self.assertEqual(ut.load_verdict(0, {}), "inconclusive") + + +class ClearScratch(unittest.TestCase): + """R-656. RED-PROOF (REPORT.md): make clear_scratch_folders return [] without removing — the first + test fails with the last run's config still there.""" + + def setUp(self): + self.root = tempfile.mkdtemp(prefix="bench-scratch-") + self.hdd = os.path.join(self.root, "hdd") + self.env = {"HDD_PATH": self.hdd, "USERDATA_PATH": self.hdd + "/userdata"} + self.said = [] + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + + def plant(self, rel): + p = os.path.join(self.hdd, rel, "config", "config.php") + os.makedirs(os.path.dirname(p), exist_ok=True) + open(p, "w").write("last run") + return p + + def test_the_apps_own_folders_are_cleared_and_said(self): + mine = self.plant("appdata/nextcloud") + other = self.plant("appdata/immich") + comp = "services:\n nextcloud:\n volumes:\n - ${HDD_PATH}/appdata/nextcloud:/var/www/html\n" + got = ut.clear_scratch_folders(comp, self.env, self.said.append) + self.assertFalse(os.path.exists(mine), "the last run's files are still there") + self.assertTrue(os.path.exists(other), "another app's folder was touched") + self.assertEqual(len(got), 1) + self.assertTrue(any("cleared before FROM" in s for s in self.said)) + + def test_never_a_bare_root_never_outside(self): + keep = self.plant("appdata/x") + comp = (" volumes:\n - ${HDD_PATH}:/data\n - ${HDD_PATH}/:/d2\n" + " - ${HDD_PATH}/../escape:/e\n") + got = ut.clear_scratch_folders(comp, self.env, self.said.append) + self.assertEqual(got, []) + self.assertTrue(os.path.exists(keep)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/upgrade-test.py b/scripts/upgrade-test.py index 03e5e2d..3f7f720 100755 --- a/scripts/upgrade-test.py +++ b/scripts/upgrade-test.py @@ -465,12 +465,50 @@ def memory_watch(app: str, project: str, workdir: Path, seconds: int, say, ev: P tight = [n for n, p in per.items() if _tight_pct(p) is not None and _tight_pct(p) > MEMORY_TIGHT] rec = {"soak_s": round(time.time() - t0, 1), "requested_s": seconds, "requests": hits["n"], "codes": hits["codes"], "first_kill": first_bad, "containers": per, - "unmeasured": [n for n, p in per.items() if not p["measured"]]} + "unmeasured": [n for n, p in per.items() if not p["measured"]], + "load": load_verdict(hits["n"], hits["codes"])} marks = ["memory_tight"] if tight and not killed else [] say(f"memory watch: killed={killed} tight={tight} requests={hits['n']} codes={hits['codes']}") return rec, killed, marks +def load_verdict(requests: int, codes: dict) -> str: + """R-653: did the watch's load REACH the app? `reached` when at least half its requests got any HTTP + answer (a 401 or a 30x is the app answering); `inconclusive` otherwise. Measured 2026-09-23 night: + ghost's and nextcloud's first ten-minute watches sent 11 797 and 9 427 requests and EVERY one was + `err` — memory measured, app idle — and the harness wrote `proven` over it.""" + if requests <= 0: + return "inconclusive" + answered = sum(n for c, n in codes.items() if c != "err") + return "reached" if answered * 2 >= requests else "inconclusive" + + +def clear_scratch_folders(compose_text: str, env: dict, say) -> list: + """R-656: before FROM, remove the app's OWN scratch drive folders — every `${HDD_PATH}/…`, + `${USERDATA_PATH}/…` and `${IMPORT_PATH}/…` bind the compose names — so a re-run of the same app does + not start on the last run's files (nextcloud's second run never installed, 2026-09-23 night: `occ + status: installed: false` over the first run's config/). `compose down -v` removes named volumes, not + bind-mounted host folders. NEVER a bare root, never a path outside the scratch roots. Returns what was + removed, and says so either way.""" + roots = {k: env.get(k) for k in ("HDD_PATH", "USERDATA_PATH", "IMPORT_PATH") if env.get(k)} + removed = [] + for var, rel in re.findall(r"\$\{(HDD_PATH|USERDATA_PATH|IMPORT_PATH)\}(/[^:\s\"']*)", compose_text): + root = roots.get(var) + if not root: + continue + rel = rel.strip("/") + target = os.path.realpath(os.path.join(root, rel)) + safe_root = os.path.realpath(root) + if not rel or target == safe_root or not target.startswith(safe_root + os.sep): + say(f"scratch folder NOT cleared (outside or equal to {var}={root}): {rel!r}") + continue + if os.path.exists(target): + shutil.rmtree(target, ignore_errors=False) + removed.append(target) + say(f"scratch drive folders cleared before FROM (R-656): {removed or 'none existed'}") + return removed + + MIGRATION_RE = re.compile( r"migrat|upgrad|schema|alter table|CREATE TABLE|InnoDB: Upgrad|mysql_upgrade|" r"mariadb-upgrade|Running .* migration|Applying|db:migrate", @@ -558,6 +596,7 @@ def run_edge(edge_id: str) -> dict: try: # --- 1. FROM --- + rec["scratch_cleared"] = clear_scratch_folders(compose_text, env, say) say(f"{edge_id}: deploying {app} at FROM {e['frm']}") render(app, e["frm"], workdir, env, e.get("template")) up = compose(workdir, project, "up", "-d") @@ -652,6 +691,11 @@ def run_edge(edge_id: str) -> dict: if killed: rec["verdict"] = "failed" say("VERDICT -> failed: the new version was OOM-killed or restarted under light load") + elif mem.get("load") != "reached": + rec["verdict"] = "inconclusive" + rec["abort_detail"] = (f"memory watch: fewer than half of its {mem.get('requests')} requests reached " + f"the app ({mem.get('codes')}) — memory measured on an idle app (R-653)") + say("VERDICT -> inconclusive: " + rec["abort_detail"]) # --- 6. the ABORT --- say(f"{edge_id}: ABORT — putting the FROM images back") @@ -778,6 +822,16 @@ def write_ladder(argv) -> int: if probs: print(f"REFUSED {app}: the entry would not be well-formed: {probs}") return 1 + # `09` §6.4 part 5: the step being SUPERSEDED keeps its own definition. When the ladder's head is the + # compose as it stands, that compose — the head's images with every fix that flowed since — becomes + # steps/.yml, the file a box one step behind will pin (check-test-record.py rule 4). + prior, _, _ = ladder.parse(fy_p.read_text()) + if prior and prior[-1].get("to") == cur: + sp = tdir / ladder.step_file(cur) + if not sp.exists(): + sp.parent.mkdir(parents=True, exist_ok=True) + sp.write_text(comp) + print(f"STEP {app}: the superseded step {cur} keeps its definition at {ladder.step_file(cur)}") # move the compose, per service, on that service's own image: line out, svc = [], None for line in comp.splitlines(): diff --git a/templates/emby/steps/59645352efe5f728.yml b/templates/emby/steps/59645352efe5f728.yml new file mode 100644 index 0000000..57c22ac --- /dev/null +++ b/templates/emby/steps/59645352efe5f728.yml @@ -0,0 +1,53 @@ +# Emby - Személyes média szerver élő TV és DVR támogatással +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: None (file-based) +# RAM: ~512M (mem_limit: 2048M) | Pi-compatible: No +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# USERDATA_PATH - Ügyfél-tartalom gyökér (/userdata) +# +# Storage layout (felhom userdata convention): +# Médiatár → ${USERDATA_PATH}/media (csak olvasható) + +services: + emby: + image: emby/embyserver:4.11.0.1 + container_name: emby + restart: unless-stopped + environment: + - TZ=Europe/Budapest + - UID=1000 + - GID=1000 + volumes: + - emby_config:/config + - ${USERDATA_PATH}/media:/media:ro + networks: + - traefik-public + deploy: + resources: + limits: + memory: 2048M + healthcheck: + # Az Emby képfájlban NINCS curl és nincs önálló wget — csak BusyBox van. + # A korábbi curl-próba ezért soha nem futott le, a konténer véglegesen + # unhealthy maradt, és a Traefik nem irányított rá forgalmat (404). + test: ["CMD", "/bin/busybox", "wget", "--spider", "-q", "http://127.0.0.1:8096/emby/system/ping"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.emby.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.emby.entrypoints=websecure" + - "traefik.http.routers.emby.tls=true" + - "traefik.http.routers.emby.tls.certresolver=letsencrypt" + - "traefik.http.services.emby.loadbalancer.server.port=8096" + +volumes: + emby_config: + +networks: + traefik-public: + external: true diff --git a/templates/ghost/steps/2ca9e0b50e635ffa.yml b/templates/ghost/steps/2ca9e0b50e635ffa.yml new file mode 100644 index 0000000..2f3d059 --- /dev/null +++ b/templates/ghost/steps/2ca9e0b50e635ffa.yml @@ -0,0 +1,47 @@ +# Ghost - Professzionális blog és hírlevél platform +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: None (file-based) +# RAM: ~150M (mem_limit: 512M) | Pi-compatible: No +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) + +services: + ghost: + image: ghost:6.64.0-alpine + container_name: ghost + restart: unless-stopped + environment: + - TZ=Europe/Budapest + - NODE_ENV=production + - url=https://${SUBDOMAIN}.${DOMAIN} + - database__client=sqlite3 + - database__connection__filename=content/data/ghost.db + volumes: + - ghost_content:/var/lib/ghost/content + networks: + - traefik-public + deploy: + resources: + limits: + memory: 512M + healthcheck: + test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:2368/',r=>{process.exit(r.statusCode<400?0:1)}).on('error',()=>process.exit(1))"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.ghost.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.ghost.entrypoints=websecure" + - "traefik.http.routers.ghost.tls=true" + - "traefik.http.routers.ghost.tls.certresolver=letsencrypt" + - "traefik.http.services.ghost.loadbalancer.server.port=2368" + +volumes: + ghost_content: + +networks: + traefik-public: + external: true diff --git a/templates/immich/steps/0b8272068aab36bf.yml b/templates/immich/steps/0b8272068aab36bf.yml new file mode 100644 index 0000000..f07d270 --- /dev/null +++ b/templates/immich/steps/0b8272068aab36bf.yml @@ -0,0 +1,146 @@ +# Immich - Self-hosted Photo & Video Management +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: PostgreSQL (with VectorChord) + Redis +# RAM: ~4GB minimum (mem_limit: 4096M total — server 2048M + ML 1536M + postgres 256M + redis 128M) | Pi-compatible: No (ML too heavy) +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# HDD_PATH - Drive namespace root (managed upload lives under appdata) +# USERDATA_PATH - Ügyfél-tartalom gyökér (/userdata) +# DB_PASSWORD - PostgreSQL password (auto-generated) +# +# Storage layout (felhom userdata convention): +# Managed upload (Immich-owned) → ${HDD_PATH}/appdata/immich (HDD, host path — app-managed) +# External photo library (RO) → ${USERDATA_PATH}/media/photos → /external/photos +# PostgreSQL data → immich_postgres_data (named volume, NVMe) +# ML model cache → immich_ml_cache (named volume, NVMe) +# Redis data → immich_redis_data (named volume, NVMe) +# +# ⚠ EXTERNAL LIBRARY IS NOT WIRED BY COMPOSE: mounting /external/photos only makes the files +# visible to the container. To actually surface them in Immich you MUST register an External +# Library pointing at /external/photos in the Immich admin UI (Administration → External +# Libraries) or via the API — a POST-DEPLOY step. Until then photos sharing is "mount ready, +# registration pending". See .felhom.yml first_steps + REPORT. +# +# First-time setup: +# Create admin account on first visit, then register the External Library (see above). + +services: + immich-server: + image: ghcr.io/immich-app/immich-server:v3.2.2 + container_name: immich-server + restart: unless-stopped + depends_on: + immich-postgres: + condition: service_healthy + immich-redis: + condition: service_healthy + environment: + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOSTNAME=immich-postgres + - DB_USERNAME=immich + - DB_DATABASE_NAME=immich + - REDIS_HOSTNAME=immich-redis + - IMMICH_MACHINE_LEARNING_URL=http://immich-machine-learning:3003 + - TZ=Europe/Budapest + volumes: + - ${HDD_PATH}/appdata/immich:/usr/src/app/upload + - ${USERDATA_PATH}/media/photos:/external/photos:ro + networks: + - traefik-public + - immich-internal + deploy: + resources: + limits: + memory: 2048M + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:2283/api/server/ping"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + labels: + - "traefik.enable=true" + - "traefik.http.routers.immich.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.immich.entrypoints=websecure" + - "traefik.http.routers.immich.tls=true" + - "traefik.http.routers.immich.tls.certresolver=letsencrypt" + - "traefik.http.services.immich.loadbalancer.server.port=2283" + + immich-machine-learning: + image: ghcr.io/immich-app/immich-machine-learning:v3.0.3 + container_name: immich-machine-learning + restart: unless-stopped + environment: + - TZ=Europe/Budapest + - TRANSFORMERS_CACHE=/cache + volumes: + - immich_ml_cache:/cache + networks: + - immich-internal + deploy: + resources: + limits: + memory: 1536M + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:3003/ping')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + + immich-postgres: + image: ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0 + container_name: immich-postgres + restart: unless-stopped + environment: + - POSTGRES_USER=immich + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=immich + - POSTGRES_INITDB_ARGS=--data-checksums + - TZ=Europe/Budapest + volumes: + - immich_postgres_data:/var/lib/postgresql/data + networks: + - immich-internal + deploy: + resources: + limits: + memory: 512M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U immich -d immich"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + immich-redis: + image: redis:7-alpine + container_name: immich-redis + restart: unless-stopped + command: redis-server --appendonly yes + environment: + - TZ=Europe/Budapest + volumes: + - immich_redis_data:/data + networks: + - immich-internal + deploy: + resources: + limits: + memory: 128M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + +volumes: + immich_ml_cache: + immich_postgres_data: + immich_redis_data: + +networks: + traefik-public: + external: true + immich-internal: diff --git a/templates/n8n/steps/9e432b2ae70c14e3.yml b/templates/n8n/steps/9e432b2ae70c14e3.yml new file mode 100644 index 0000000..c91b7dc --- /dev/null +++ b/templates/n8n/steps/9e432b2ae70c14e3.yml @@ -0,0 +1,48 @@ +# n8n - Workflow automatizálás vizuális szerkesztővel +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: None (file-based) +# RAM: ~150M (mem_limit: 512M) | Pi-compatible: No +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# N8N_ENCRYPTION_KEY- Titkosítási kulcs (auto-generated) + +services: + n8n: + image: n8nio/n8n:2.40.5 + container_name: n8n + restart: unless-stopped + environment: + - TZ=Europe/Budapest + - N8N_HOST=${SUBDOMAIN}.${DOMAIN} + - N8N_PROTOCOL=https + - WEBHOOK_URL=https://${SUBDOMAIN}.${DOMAIN}/ + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + volumes: + - n8n_data:/home/node/.n8n + networks: + - traefik-public + deploy: + resources: + limits: + memory: 1536M + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:5678/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.n8n.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.n8n.entrypoints=websecure" + - "traefik.http.routers.n8n.tls=true" + - "traefik.http.routers.n8n.tls.certresolver=letsencrypt" + - "traefik.http.services.n8n.loadbalancer.server.port=5678" + +volumes: + n8n_data: + +networks: + traefik-public: + external: true diff --git a/templates/navidrome/steps/755d09b083f757a0.yml b/templates/navidrome/steps/755d09b083f757a0.yml new file mode 100644 index 0000000..97db6c5 --- /dev/null +++ b/templates/navidrome/steps/755d09b083f757a0.yml @@ -0,0 +1,51 @@ +# Navidrome - Könnyű zene szerver Subsonic API támogatással +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: None (file-based) +# RAM: ~50M (mem_limit: 256M) | Pi-compatible: Yes +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# USERDATA_PATH - Ügyfél-tartalom gyökér (/userdata) +# +# Storage layout (felhom userdata convention): +# Zenegyűjtemény → ${USERDATA_PATH}/media/music (csak olvasható) + +services: + navidrome: + image: deluan/navidrome:0.64.0 + container_name: navidrome + restart: unless-stopped + environment: + - TZ=Europe/Budapest + - ND_SCANSCHEDULE=1h + - ND_LOGLEVEL=info + - ND_BASEURL= + volumes: + - navidrome_data:/data + - ${USERDATA_PATH}/media/music:/music:ro + networks: + - traefik-public + deploy: + resources: + limits: + memory: 256M + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:4533/ping"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.navidrome.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.navidrome.entrypoints=websecure" + - "traefik.http.routers.navidrome.tls=true" + - "traefik.http.routers.navidrome.tls.certresolver=letsencrypt" + - "traefik.http.services.navidrome.loadbalancer.server.port=4533" + +volumes: + navidrome_data: + +networks: + traefik-public: + external: true diff --git a/templates/nextcloud/steps/eef2e4afe1218021.yml b/templates/nextcloud/steps/eef2e4afe1218021.yml new file mode 100644 index 0000000..e546d9a --- /dev/null +++ b/templates/nextcloud/steps/eef2e4afe1218021.yml @@ -0,0 +1,132 @@ +# Nextcloud - Saját felhő tárhely - Google Drive/Dropbox alternatíva +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: mariadb +# RAM: ~256M (mem_limit: 1024M) | Pi-compatible: No +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# DB_PASSWORD - Adatbázis jelszó (auto-generated) +# MYSQL_ROOT_PASSWORD- MariaDB root jelszó (auto-generated) +# NEXTCLOUD_ADMIN_USER- Admin felhasználónév +# NEXTCLOUD_ADMIN_PASSWORD- Admin jelszó (auto-generated) +# HDD_PATH - Adattárolási útvonal + +services: + nextcloud: + image: nextcloud:34.0.1-apache + container_name: nextcloud + restart: unless-stopped + depends_on: + nextcloud-db: + condition: service_healthy + nextcloud-redis: + condition: service_healthy + environment: + - TZ=Europe/Budapest + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + - MYSQL_PASSWORD=${DB_PASSWORD} + - MYSQL_HOST=nextcloud-db + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER:-admin} + - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - NEXTCLOUD_TRUSTED_DOMAINS=${SUBDOMAIN}.${DOMAIN} nextcloud + - OVERWRITEPROTOCOL=https + - OVERWRITEHOST=${SUBDOMAIN}.${DOMAIN} + - REDIS_HOST=nextcloud-redis + # App-email (managed relay). Injected by the controller only when app-email is on (global + per-app); + # empty SMTP_HOST keeps Nextcloud mail disabled. Nextcloud uses the plaintext :2526 listener + # (tls_mode=plaintext, SMTP_SECURE empty = no TLS) — it can't skip the self-signed STARTTLS cert. + # From is split: MAIL_FROM_ADDRESS=nextcloud + MAIL_DOMAIN=felhom.eu. See .felhom.yml smtp_mapping. + - SMTP_HOST=${SMTP_HOST:-} + - SMTP_PORT=${SMTP_PORT:-25} + - SMTP_SECURE=${SMTP_SECURE:-} + - MAIL_FROM_ADDRESS=${MAIL_FROM_ADDRESS:-} + - MAIL_DOMAIN=${MAIL_DOMAIN:-} + volumes: + - nextcloud_html:/var/www/html + - ${HDD_PATH}/appdata/nextcloud:/var/www/html/data + networks: + - traefik-public + - nextcloud-internal + deploy: + resources: + limits: + memory: 1024M + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:80/status.php"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.nextcloud.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.nextcloud.entrypoints=websecure" + - "traefik.http.routers.nextcloud.tls=true" + - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt" + - "traefik.http.services.nextcloud.loadbalancer.server.port=80" + - "traefik.http.middlewares.nextcloud-redirect.redirectregex.regex=/.well-known/(card|cal)dav" + - "traefik.http.middlewares.nextcloud-redirect.redirectregex.replacement=/remote.php/dav/" + - "traefik.http.routers.nextcloud.middlewares=nextcloud-redirect" + + nextcloud-db: + image: mariadb:12.3 + container_name: nextcloud-db + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + - MYSQL_PASSWORD=${DB_PASSWORD} + - TZ=Europe/Budapest + # MARIADB_AUTO_UPGRADE: on a MAJOR engine move the engine converts its own datadir (~7 s on a + # small DB, backs its system tables up first). Operator ruling 2026-09-13 on + # felhom.eu/documentation/audits/SPIKE-r459-mariadb-upgrade-2026-09-06.md. Inert until a + # major moves — and none may, until Slice 4 (R-448) ships: see CLAUDE.md, engine-major rule. + - MARIADB_AUTO_UPGRADE=1 + volumes: + - nextcloud_db_data:/var/lib/mysql + networks: + - nextcloud-internal + deploy: + resources: + limits: + memory: 512M + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + + nextcloud-redis: + image: redis:7-alpine + container_name: nextcloud-redis + restart: unless-stopped + command: redis-server --appendonly yes + environment: + - TZ=Europe/Budapest + volumes: + - nextcloud_redis_data:/data + networks: + - nextcloud-internal + deploy: + resources: + limits: + memory: 128M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + +volumes: + nextcloud_db_data: + nextcloud_html: + nextcloud_redis_data: + +networks: + traefik-public: + external: true + nextcloud-internal: diff --git a/templates/romm/steps/7f9c6a74b5891f8e.yml b/templates/romm/steps/7f9c6a74b5891f8e.yml new file mode 100644 index 0000000..a201e38 --- /dev/null +++ b/templates/romm/steps/7f9c6a74b5891f8e.yml @@ -0,0 +1,160 @@ +# ROMM - ROM Manager for Game Libraries +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: MariaDB + Redis +# RAM: ~300MB (mem_limit: 1280M total — romm 768M + mariadb 384M + redis 128M) | Pi-compatible: No (MariaDB + heavy) +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# HDD_PATH - Drive namespace root (appdata lives here) +# USERDATA_PATH - Ügyfél-tartalom gyökér (/userdata) +# DB_PASSWORD - MariaDB user password (auto-generated) +# MYSQL_ROOT_PASSWORD - MariaDB root password (auto-generated) +# ROMM_AUTH_SECRET_KEY - Auth secret (auto-generated) +# +# Storage layout (felhom userdata convention): +# ROM library → ${USERDATA_PATH}/roms (browsable — drop ROMs here via FileBrowser) +# Cover art etc → ${HDD_PATH}/appdata/romm/resources (app-internal, NOT browsable) +# App config → romm_config (named volume, NVMe) +# MariaDB data → romm_db_data (named volume, NVMe) +# Redis data → romm_redis_data (named volume, NVMe) +# +# First-time setup: +# Default login: admin / admin — change immediately! + +services: + romm: + image: rommapp/romm:5.3.0 + container_name: romm + restart: unless-stopped + depends_on: + romm-db: + condition: service_healthy + romm-redis: + condition: service_healthy + entrypoint: ["/bin/sh", "-c"] + command: + - | + if [ ! -f /romm/config/config.yml ]; then + echo "Creating default config.yml..." + cat > /romm/config/config.yml << 'CONF' + exclude: + platforms: [] + roms: [] + system: + log_level: INFO + CONF + fi + exec /docker-entrypoint.sh /init + environment: + - ROMM_AUTH_SECRET_KEY=${ROMM_AUTH_SECRET_KEY} + - DB_PASSWD=${DB_PASSWORD} + - DB_HOST=romm-db + - DB_PORT=3306 + - DB_NAME=romm + - DB_USER=romm + - REDIS_HOST=romm-redis + - REDIS_PORT=6379 + - ROMM_PORT=8080 + # 2, not the image's default of 4. MEASURED on demo-hp 2026-09-22 (R-635): each warm + # uvicorn worker holds ~216 MiB, so four of them plus the master reach ~882 MiB and the + # container was OOM-killed at both 512M and 768M — 37 worker SIGKILLs in five minutes, + # ~500% CPU, host load 5.2 while otherwise idle. Four workers is a SERVER default; this + # is one household on one small box. Two workers measure ~450 MiB and fit 768M with + # real headroom. `/init:143` reads this variable: --workers "${WEB_SERVER_CONCURRENCY:-4}". + - WEB_SERVER_CONCURRENCY=2 + - IGDB_CLIENT_ID=${IGDB_CLIENT_ID:-} + - IGDB_CLIENT_SECRET=${IGDB_CLIENT_SECRET:-} + - STEAMGRIDDB_API_KEY=${STEAMGRIDDB_API_KEY:-} + - SCREENSCRAPER_USER=${SCREENSCRAPER_USER:-} + - SCREENSCRAPER_PASSWORD=${SCREENSCRAPER_PASSWORD:-} + - MOBYGAMES_API_KEY=${MOBYGAMES_API_KEY:-} + - TZ=Europe/Budapest + volumes: + - ${USERDATA_PATH}/roms:/romm/library + - ${HDD_PATH}/appdata/romm/resources:/romm/resources + - romm_config:/romm/config + networks: + - traefik-public + - romm-internal + deploy: + resources: + limits: + # 768M, not 512M: romm 5.3.0 does not fit in 512M. Measured on demo-hp 2026-09-22 — + # OOMKilled true, 4,530 gunicorn worker SIGKILLs in six hours, ~500% CPU in a permanent + # restart storm, while the web front end still answered 200 so the update read `done` + # (R-635). 5.0.0 did fit; the version moved and the limit did not. + memory: 768M + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8080/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + labels: + - "traefik.enable=true" + - "traefik.http.routers.romm.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.romm.entrypoints=websecure" + - "traefik.http.routers.romm.tls=true" + - "traefik.http.routers.romm.tls.certresolver=letsencrypt" + - "traefik.http.services.romm.loadbalancer.server.port=8080" + + romm-db: + image: mariadb:11.4 + container_name: romm-db + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=romm + - MYSQL_USER=romm + - MYSQL_PASSWORD=${DB_PASSWORD} + - TZ=Europe/Budapest + # MARIADB_AUTO_UPGRADE: on a MAJOR engine move the engine converts its own datadir (~7 s on a + # small DB, backs its system tables up first). Operator ruling 2026-09-13 on + # felhom.eu/documentation/audits/SPIKE-r459-mariadb-upgrade-2026-09-06.md. Inert until a + # major moves — and none may, until Slice 4 (R-448) ships: see CLAUDE.md, engine-major rule. + - MARIADB_AUTO_UPGRADE=1 + volumes: + - romm_db_data:/var/lib/mysql + networks: + - romm-internal + deploy: + resources: + limits: + memory: 384M + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + romm-redis: + image: redis:7-alpine + container_name: romm-redis + restart: unless-stopped + command: redis-server --appendonly yes + environment: + - TZ=Europe/Budapest + volumes: + - romm_redis_data:/data + networks: + - romm-internal + deploy: + resources: + limits: + memory: 128M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + +volumes: + romm_config: + romm_db_data: + romm_redis_data: + +networks: + traefik-public: + external: true + romm-internal: diff --git a/templates/romm/steps/90dd9d68258286ef.yml b/templates/romm/steps/90dd9d68258286ef.yml new file mode 100644 index 0000000..dffeb31 --- /dev/null +++ b/templates/romm/steps/90dd9d68258286ef.yml @@ -0,0 +1,160 @@ +# ROMM - ROM Manager for Game Libraries +# Domain: ${SUBDOMAIN}.${DOMAIN} +# Database: MariaDB + Redis +# RAM: ~300MB (mem_limit: 1280M total — romm 768M + mariadb 384M + redis 128M) | Pi-compatible: No (MariaDB + heavy) +# +# Environment variables: +# DOMAIN - Your domain (e.g., demo-felhom.eu) +# HDD_PATH - Drive namespace root (appdata lives here) +# USERDATA_PATH - Ügyfél-tartalom gyökér (/userdata) +# DB_PASSWORD - MariaDB user password (auto-generated) +# MYSQL_ROOT_PASSWORD - MariaDB root password (auto-generated) +# ROMM_AUTH_SECRET_KEY - Auth secret (auto-generated) +# +# Storage layout (felhom userdata convention): +# ROM library → ${USERDATA_PATH}/roms (browsable — drop ROMs here via FileBrowser) +# Cover art etc → ${HDD_PATH}/appdata/romm/resources (app-internal, NOT browsable) +# App config → romm_config (named volume, NVMe) +# MariaDB data → romm_db_data (named volume, NVMe) +# Redis data → romm_redis_data (named volume, NVMe) +# +# First-time setup: +# Default login: admin / admin — change immediately! + +services: + romm: + image: rommapp/romm:5.3.1 + container_name: romm + restart: unless-stopped + depends_on: + romm-db: + condition: service_healthy + romm-redis: + condition: service_healthy + entrypoint: ["/bin/sh", "-c"] + command: + - | + if [ ! -f /romm/config/config.yml ]; then + echo "Creating default config.yml..." + cat > /romm/config/config.yml << 'CONF' + exclude: + platforms: [] + roms: [] + system: + log_level: INFO + CONF + fi + exec /docker-entrypoint.sh /init + environment: + - ROMM_AUTH_SECRET_KEY=${ROMM_AUTH_SECRET_KEY} + - DB_PASSWD=${DB_PASSWORD} + - DB_HOST=romm-db + - DB_PORT=3306 + - DB_NAME=romm + - DB_USER=romm + - REDIS_HOST=romm-redis + - REDIS_PORT=6379 + - ROMM_PORT=8080 + # 2, not the image's default of 4. MEASURED on demo-hp 2026-09-22 (R-635): each warm + # uvicorn worker holds ~216 MiB, so four of them plus the master reach ~882 MiB and the + # container was OOM-killed at both 512M and 768M — 37 worker SIGKILLs in five minutes, + # ~500% CPU, host load 5.2 while otherwise idle. Four workers is a SERVER default; this + # is one household on one small box. Two workers measure ~450 MiB and fit 768M with + # real headroom. `/init:143` reads this variable: --workers "${WEB_SERVER_CONCURRENCY:-4}". + - WEB_SERVER_CONCURRENCY=2 + - IGDB_CLIENT_ID=${IGDB_CLIENT_ID:-} + - IGDB_CLIENT_SECRET=${IGDB_CLIENT_SECRET:-} + - STEAMGRIDDB_API_KEY=${STEAMGRIDDB_API_KEY:-} + - SCREENSCRAPER_USER=${SCREENSCRAPER_USER:-} + - SCREENSCRAPER_PASSWORD=${SCREENSCRAPER_PASSWORD:-} + - MOBYGAMES_API_KEY=${MOBYGAMES_API_KEY:-} + - TZ=Europe/Budapest + volumes: + - ${USERDATA_PATH}/roms:/romm/library + - ${HDD_PATH}/appdata/romm/resources:/romm/resources + - romm_config:/romm/config + networks: + - traefik-public + - romm-internal + deploy: + resources: + limits: + # 768M, not 512M: romm 5.3.0 does not fit in 512M. Measured on demo-hp 2026-09-22 — + # OOMKilled true, 4,530 gunicorn worker SIGKILLs in six hours, ~500% CPU in a permanent + # restart storm, while the web front end still answered 200 so the update read `done` + # (R-635). 5.0.0 did fit; the version moved and the limit did not. + memory: 768M + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8080/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + labels: + - "traefik.enable=true" + - "traefik.http.routers.romm.rule=Host(`${SUBDOMAIN}.${DOMAIN}`)" + - "traefik.http.routers.romm.entrypoints=websecure" + - "traefik.http.routers.romm.tls=true" + - "traefik.http.routers.romm.tls.certresolver=letsencrypt" + - "traefik.http.services.romm.loadbalancer.server.port=8080" + + romm-db: + image: mariadb:11.4 + container_name: romm-db + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=romm + - MYSQL_USER=romm + - MYSQL_PASSWORD=${DB_PASSWORD} + - TZ=Europe/Budapest + # MARIADB_AUTO_UPGRADE: on a MAJOR engine move the engine converts its own datadir (~7 s on a + # small DB, backs its system tables up first). Operator ruling 2026-09-13 on + # felhom.eu/documentation/audits/SPIKE-r459-mariadb-upgrade-2026-09-06.md. Inert until a + # major moves — and none may, until Slice 4 (R-448) ships: see CLAUDE.md, engine-major rule. + - MARIADB_AUTO_UPGRADE=1 + volumes: + - romm_db_data:/var/lib/mysql + networks: + - romm-internal + deploy: + resources: + limits: + memory: 384M + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + romm-redis: + image: redis:7-alpine + container_name: romm-redis + restart: unless-stopped + command: redis-server --appendonly yes + environment: + - TZ=Europe/Budapest + volumes: + - romm_redis_data:/data + networks: + - romm-internal + deploy: + resources: + limits: + memory: 128M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + +volumes: + romm_config: + romm_db_data: + romm_redis_data: + +networks: + traefik-public: + external: true + romm-internal: