security watch reads zstd layers in CI by itself (R-1327): pinned zstandard 0.25.0 wheel, sha256-checked; RUNTIME log line; HA 2026.9.4 and RomM 5.4.1 dropped from the cache for the CI fill proof; Immich v3.3.5 power-cut repair evidence

2026-10-28 07:59:34 +01:00
1 parent 9b4b5fe5e7
commit a3247d9f44
@@ -691,9 +691,12 @@ needed the layers — Alpine/Debian package databases, Node's `node_version.h`,
 matches it against `scripts/security_watch_runtimes.json`; a hit is an ADVISORY with source „runtime", a read that
 fails is NOT CHECKED. Results are cached by the tag's digest in `state.json` (`runtime_cache`) and the nightly lookup is
 a HEAD request (Docker Hub does not count it against the anonymous pull limit — the first run's GETs hit HTTP 429).
-**Decided by CC unattended 2026-10-27 — operator may reverse:** the CI runner has no zstd (Python 3.13 has none), so
-images with zstd layers (Home Assistant) are read on DooPlex by a filling run (`--no-mail`, the stored Docker Hub login
-through `sec run`) and committed into the cache; CI reads them from there and lists a new unread zstd image as NOT
-CHECKED — instead of a new dependency in CI. The DooPlex group (`documentation/security-watch/dooplex-workloads.txt`,
+**The cache fills itself in CI (2026-10-28, R-1327; replaces the 2026-10-27 night's DooPlex filling run).** Images with
+zstd layers (Home Assistant) are read with the `zstandard` package, version 0.25.0, as a wheel pinned by the sha256 PyPI
+publishes and checked every time it is opened; the workflows fetch it (`FELHOM_WHEEL_FETCH=1`) into the runner's cache.
+Not the `zstd` program in the CI runner image: that image is built in the homelab repo, and changing it is a DooPlex
+change. A missing or wrong wheel makes the zstd layers NOT CHECKED, said in the log's `RUNTIME` line — never a pass. A
+new image is read on the night its tag first appears; the log's `RUNTIME read=… (… s) from-cache=…` line says how much
+was read and how long it took. The DooPlex group (`documentation/security-watch/dooplex-workloads.txt`,
 from homelab-manifests) is read the same way and mailed in its own block; Felhom changes nothing on DooPlex.

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""immich_repair.py EVDIR — re-prove Immich's after_unclean_boot scripts on v3.3.5 on scratch 9202 (the method of
+felhom.eu audits/immich-repair-2026-10-14 Part C): deploy through the product (9202 pointed at the drill, Immich
+v3.3.5), the admin through the setup gate, a family member, 20 + 10 photos through Immich's own API, then damage
+4 thumbnail files and delete 1 original, run the template's repair and report_lost commands inside immich-server
+exactly as the controller does (node -e <script> <ids…>), and check: 0 empty files after, 0 controls rewritten,
+the deleted original reported by name. Remove through the product at the end. No password is printed."""
+import hashlib, io, json, os, re, sys, tempfile, time
+
+sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
+import box_walk as w                      # noqa: E402
+import upgrade_fixtures_box28 as fx       # noqa: E402
+import yaml                               # noqa: E402
+from PIL import Image                     # noqa: E402
+
+EV = sys.argv[1]
+os.makedirs(EV, exist_ok=True)
+DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
+OUT = []
+
+
+def say(*a):
+    w.say(*a)
+    OUT.append(w.LOG[-1])
+
+
+def save():
+    open(os.path.join(EV, "repair-v3.3.5.txt"), "w").write("\n".join(OUT) + "\n")
+
+
+def jpeg(i):
+    im = Image.new("RGB", (640, 480), ((i * 37) % 256, (i * 91) % 256, (i * 53) % 256))
+    for y in range(0, 480, 16):
+        for x in range(0, 640, 16):
+            if (x // 16 + y // 16 + i) % 3 == 0:
+                im.paste(((x * i) % 256, (y * 7) % 256, (i * 13) % 256), (x, y, x + 16, y + 16))
+    b = io.BytesIO()
+    im.save(b, "JPEG", quality=85)
+    return b.getvalue()
+
+
+def login(sub, u, pw):
+    rc, code, out = w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json",
+                               data=json.dumps({"email": u, "password": pw}), method="POST")
+    return json.loads(out)["accessToken"]
+
+
+def upload(sub, tok, n, tag, tmp):
+    ok = 0
+    for i in range(n):
+        p = os.path.join(tmp, f"{tag}-{i:03d}.jpg")
+        open(p, "wb").write(jpeg(i + (100 if tag == "mem" else 0)))
+        ts = "2026-01-%02dT%02d:00:00.000Z" % (1 + i % 28, i % 24)
+        rc, code, out = w.app_curl(sub, "/api/assets", "-H", f"Authorization: Bearer {tok}",
+                                   "-F", f"assetData=@{p};type=image/jpeg", "-F", f"deviceAssetId={tag}-{i}",
+                                   "-F", "deviceId=felhom-drill", "-F", f"fileCreatedAt={ts}", "-F", f"fileModifiedAt={ts}",
+                                   method="POST", timeout=90)
+        if code in ("200", "201"):
+            ok += 1
+        else:
+            say(f"  upload {tag} {i} -> {code} {out[:150]}")
+    return ok
+
+
+def node_cmd(block):
+    cmd = block["command"]
+    assert cmd[:2] == ["node", "-e"], cmd
+    return cmd[2]
+
+
+def main():
+    fy = yaml.safe_load(open(f"{DRILL}/templates/immich/.felhom.yml"))
+    aub = fy["after_unclean_boot"]
+    comp = open(f"{DRILL}/templates/immich/docker-compose.yml").read()
+    img = re.search(r"image:\s*(ghcr.io/immich-app/immich-server:\S+)", comp).group(1)
+    say(f"drill head {os.popen(f'git -C {DRILL} rev-parse --short HEAD').read().strip()}: immich-server {img}; "
+        f"the block lists {aub['images']}")
+    w.login()
+    if w.stack("immich").get("deployed"):
+        say("immich already deployed on 9202 — STOP (not ours)")
+        return 2
+    if not w.deploy("immich", "photos"):
+        return 1
+    pins = (w.stack("immich").get("app_config") or {}).get("pinned_images") or {}
+    say(f"deployed: {pins.get('immich-server')}")
+    if pins.get("immich-server") != img:
+        say("the box did not deploy the drill's v3.3.5 — STOP")
+        return 1
+    seed = fx.Immich().seed(w, "photos", say)
+    if not seed:
+        say("admin sign-up failed")
+        return 1
+    atok = login("photos", seed["u"], seed["pw"])
+    mu, mpw = "member" + os.urandom(3).hex() + "@example.invalid", "Drill-" + os.urandom(8).hex()
+    rc, code, out = w.app_curl("photos", "/api/admin/users", "-H", f"Authorization: Bearer {atok}",
+                               "-H", "Content-Type: application/json",
+                               data=json.dumps({"email": mu, "password": mpw, "name": "Member"}), method="POST")
+    say(f"family member: {code}")
+    mtok = login("photos", mu, mpw)
+    tmp = tempfile.mkdtemp()
+    say(f"admin uploaded: {upload('photos', atok, 20, 'adm', tmp)}")
+    say(f"member uploaded: {upload('photos', mtok, 10, 'mem', tmp)}")
+    hdd = w.app_env("immich", "HDD_PATH")
+    under = aub["empty_files"]["under"].replace("${HDD_PATH}", hdd)
+    up = under.rsplit("/thumbs", 1)[0]
+    for _ in range(60):
+        n = w.guest(f"find {under} -type f -name '*_thumbnail.webp' -size +0 | wc -l; "
+                    f"find {under} -type f -name '*_preview.jpeg' -size +0 | wc -l").split()
+        if n == ["30", "30"]:
+            break
+        time.sleep(5)
+    say(f"thumbnails/previews settled: {n}")
+    srv = w.guest("docker ps --filter label=com.docker.compose.service=immich-server --format '{{.Names}}'").strip()
+    # controls: every thumbs file before the damage (size + mtime)
+    ctl0 = w.guest(f"find {under} -type f -printf '%p %s %T@\\n' | sort")
+    files = [l.split()[0] for l in ctl0.splitlines() if l.endswith(tuple("0123456789")) and "_thumbnail.webp" in l]
+    by_user = {}
+    for f in files:
+        by_user.setdefault(f.split("/thumbs/")[1].split("/")[0], []).append(f)
+    users = sorted(by_user)
+    victims = by_user[users[0]][:2] + by_user[users[1]][:2]
+    rx = re.compile(aub["empty_files"]["id_from_name"])
+    ids = [rx.search(os.path.basename(v)).group(1) for v in victims]
+    # one original deleted (not a victim's), as the power cut can lose a file that was being written
+    orig = w.guest(f"find {up}/upload {up}/library -type f -name '*.jpg' 2>/dev/null | head -1").strip()
+    w.guest("\n".join(f": > {v}" for v in victims) + f"\nrm -f {orig}\n")
+    empty0 = w.guest(f"find {under} -type f -size 0 | wc -l").strip()
+    say(f"damage: {empty0} empty files ({len(users)} users), 1 original deleted")
+    # the repair, exactly as the controller runs it: node -e <script> <ids…> inside the service's container
+    t = f"/tmp/felhom-aub-{os.getpid()}.js"
+    rep = node_cmd(aub["repair"])
+    out = w.guest(f"cat > {t} <<'FELHOMJS'\n{rep}\nFELHOMJS\nt0=$(date +%s%N)\n"
+                  f"docker exec {srv} node -e \"$(cat {t})\" {' '.join(ids)}; echo \"rc=$? in $(( ($(date +%s%N) - t0) / 1000000 )) ms\"\n"
+                  f"rm -f {t}", timeout=900)
+    say("== repair with %d ids\n%s" % (len(ids), out.strip()))
+    empty1 = w.guest(f"find {under} -type f -size 0 | wc -l").strip()
+    ctl1 = w.guest(f"find {under} -type f -printf '%p %s %T@\\n' | sort")
+    a = {l.split()[0]: l for l in ctl0.splitlines() if l.strip()}
+    b = {l.split()[0]: l for l in ctl1.splitlines() if l.strip()}
+    # a CONTROL is a file of an asset that was NOT damaged; the job rebuilds a damaged asset's thumbnail AND preview,
+    # so its sibling files are expected to change (the 2026-10-14 method counted the same way)
+    controls = [p for p in a if not any(os.path.basename(p).startswith(i) for i in ids) and not p.endswith("/.immich")]
+    rewritten = [p for p in controls if b.get(p) != a[p]]
+    sib = [os.path.basename(p) for p in a if p not in victims and any(os.path.basename(p).startswith(i) for i in ids)]
+    say(f"damaged assets' other files (rebuilt by the job, expected): {len(sib)}; changed: "
+        f"{sum(1 for p in a if os.path.basename(p) in sib and b.get(p) != a[p])}")
+    if rewritten:
+        say("controls rewritten: " + ", ".join(os.path.basename(p) for p in rewritten))
+    fixed = [v for v in victims if v in b and b[v].split()[1] != "0"]
+    say(f"after: empty now {empty1}; victims repaired {len(fixed)} of {len(victims)}; "
+        f"controls rewritten {len(rewritten)} of {len(controls)}")
+    lost = node_cmd(aub["report_lost"])
+    out = w.guest(f"cat > {t} <<'FELHOMJS'\n{lost}\nFELHOMJS\n"
+                  f"docker exec {srv} node -e \"$(cat {t})\"; echo \"rc=$?\"\nrm -f {t}", timeout=900)
+    say("== report_lost (the deleted original: %s)\n%s" % (os.path.basename(orig), out.strip()))
+    verdict = (empty1 == "0" and len(fixed) == len(victims) and not rewritten and "FELHOM_REPAIR_OK" in "\n".join(OUT)
+               and "FELHOM_LOST_OK lost=1" in out)
+    say(f"VERDICT {'proven' if verdict else 'NOT proven'} on {img}")
+    save()
+    return 0 if verdict else 1
+
+
+if __name__ == "__main__":
+    rc = 1
+    try:
+        rc = main()
+    finally:
+        try:
+            if w.stack("immich").get("deployed"):
+                w.remove("immich")
+                OUT.extend(w.LOG[-3:])
+        finally:
+            save()
+    sys.exit(rc)
@@ -0,0 +1,20 @@
+07:49:39 drill head 34b14d5: immich-server ghcr.io/immich-app/immich-server:v3.3.5; the block lists ['ghcr.io/immich-app/immich-server:v3.2.4']
+07:51:25 deployed: ghcr.io/immich-app/immich-server:v3.3.5
+07:51:25     immich: seeded album drillalbumf9eea650
+07:51:25 family member: 201
+07:51:27 admin uploaded: 20
+07:51:27 member uploaded: 10
+07:51:44 thumbnails/previews settled: ['30', '30']
+07:52:08 damage: 4 empty files (2 users), 1 original deleted
+07:52:14 == repair with 4 ids
+FELHOM_REPAIR_OK queued=4 waited_ms=2003
+rc=0 in 2348 ms
+07:52:21 after: empty now 0; victims repaired 4 of 4; controls rewritten 4 of 56
+07:52:27 == report_lost (the deleted original: e9bab540-26d7-46dd-9e92-5a7738ffb99e.jpg)
+LOST {"name":"adm-006.jpg","taken":"2026-01-07 07:00"}
+FELHOM_LOST_OK lost=1 waited_ms=2035
+rc=0
+07:52:27 VERDICT NOT proven on ghcr.io/immich-app/immich-server:v3.3.5
+07:52:29   [X] stop -> 200 {'ok': True, 'message': 'Stack immich stop completed'}
+07:53:01   [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'immich', 'volumes_removed': ['immich_immich_ml_cache', 'immich_immich_postgres_data'], 'hdd_paths_removed': ['/mnt/felhom-driv
+07:53:10   [X] after remove: deployed=False leftovers=''
@@ -0,0 +1,21 @@
+07:53:25 drill head 34b14d5: immich-server ghcr.io/immich-app/immich-server:v3.3.5; the block lists ['ghcr.io/immich-app/immich-server:v3.2.4']
+07:54:56 deployed: ghcr.io/immich-app/immich-server:v3.3.5
+07:54:57     immich: seeded album drillalbum5cd41f9e
+07:54:58 family member: 201
+07:54:59 admin uploaded: 20
+07:55:00 member uploaded: 10
+07:55:15 thumbnails/previews settled: ['30', '30']
+07:55:37 damage: 4 empty files (2 users), 1 original deleted
+07:55:43 == repair with 4 ids
+FELHOM_REPAIR_OK queued=4 waited_ms=2004
+rc=0 in 2592 ms
+07:55:52 damaged assets' other files (rebuilt by the job, expected): 4; changed: 4
+07:55:52 after: empty now 0; victims repaired 4 of 4; controls rewritten 0 of 52
+07:55:59 == report_lost (the deleted original: 5b2bafda-860a-422b-b980-6b34049494c0.jpg)
+LOST {"name":"adm-016.jpg","taken":"2026-01-17 17:00"}
+FELHOM_LOST_OK lost=1 waited_ms=3045
+rc=0
+07:55:59 VERDICT proven on ghcr.io/immich-app/immich-server:v3.3.5
+07:56:01   [X] stop -> 200 {'ok': True, 'message': 'Stack immich stop completed'}
+07:56:34   [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'immich', 'volumes_removed': ['immich_immich_ml_cache', 'immich_immich_postgres_data'], 'hdd_paths_removed': ['/mnt/felhom-driv
+07:56:42   [X] after remove: deployed=False leftovers=''
@@ -0,0 +1,15 @@
+mirror main 34b14d50
+
+26:    repo_url: file:///opt/docker/felhom-controller/drill-catalog.git
+
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated home-assistant/.felhom.yml
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated immich/docker-compose.yml
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated immich/.felhom.yml
+2026/10/28 06:49:04 sync.go:261: [INFO] [sync] catalog moved b5b78f61cf7e -> 34b14d5003f5 (0 new, 2 stack dir(s) changed)
+2026/10/28 06:49:04 sync.go:299: [INFO] [sync] Catalog sync complete
+2026/10/28 06:49:04 sync.go:142: [INFO] [sync] Initial sync: Sablonok frissítve — frissítve: home-assistant, immich
+
+  expect n8n/docker-compose.yml ∋ N8N_DIAGNOSTICS_ENABLED=false: 1
+  expect opengist/docker-compose.yml ∋ 256M: 2
+  expect homepage/.felhom.yml ∋ family_gate: true: 1
+drill main 34b14d5, box cache 34b14d5: DRILL-CONFIRMED
@@ -0,0 +1,16 @@
+RESTORED-IDENTICAL
+26:    repo_url: https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated home-assistant/.felhom.yml
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated immich/docker-compose.yml
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated immich/.felhom.yml
+2026/10/28 06:57:05 sync.go:261: [INFO] [sync] catalog moved 34b14d5003f5 -> b5b78f61cf7e (0 new, 2 stack dir(s) changed)
+2026/10/28 06:57:05 sync.go:299: [INFO] [sync] Catalog sync complete
+2026/10/28 06:57:05 sync.go:142: [INFO] [sync] Initial sync: Sablonok frissítve — frissítve: home-assistant, immich
+
+26:    repo_url: https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+-rw------- 1 root root 1994 Oct 22 14:47 /var/lib/docker/volumes/felhom-controller-data/_data/controller.yaml.pre-release1026-drill
+mirror main 34b14d50
+cache HEAD b5b78f6
+https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+
@@ -2565,19 +2565,6 @@
   "python": null,
   "unchecked": []
  },
-  "sha256:3e6710a7ab2a61311d9d899b719f6c3657791c63e8f4942cec4ebc42401d6b76": {
-   "apk": {
-    "brotli-libs": "1.2.0-r1"
-   },
-   "brotli": "1.2.0",
-   "brotlicffi": null,
-   "digest": "sha256:e47c978e1b801466e7f62f612fd552bc3a228e077b31a3f1c22c05cf63d754da",
-   "dpkg": {},
-   "env": {},
-   "node": null,
-   "python": null,
-   "unchecked": []
-  },
  "sha256:40c454d74ffdd9ecb15833e4d96f3a630d59aad862a1869381ae6ee086454419": {
   "apk": {},
   "brotli": null,
@@ -3107,20 +3094,6 @@
   "python": null,
   "unchecked": []
  },
-  "sha256:8677fb7b81b1d3db20b849e989a1dadc2a11159e51eb34a2acdcb01102274edf": {
-   "apk": {
-    "brotli-libs": "1.2.1-r0",
-    "nodejs": "24.18.1-r0"
-   },
-   "brotli": null,
-   "brotlicffi": null,
-   "digest": "sha256:923a808be529a834a171b3e045821aeb7e2851f4b6fc9514076785b68a9142f6",
-   "dpkg": {},
-   "env": {},
-   "node": "24.18.1",
-   "python": null,
-   "unchecked": []
-  },
  "sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef": {
   "apk": {
    "brotli-libs": "1.2.0-r1"