security watch reads zstd layers in CI by itself (R-1327): pinned zstandard 0.25.0 wheel, sha256-checked; RUNTIME log line; HA 2026.9.4 and RomM 5.4.1 dropped from the cache for the CI fill proof; Immich v3.3.5 power-cut repair evidence

2026-10-28 07:59:34 +01:00
1 parent 9b4b5fe5e7
commit a3247d9f44
@@ -211,6 +211,9 @@ jobs:
          # R-986: the security-scan gate fetches the PINNED gitleaks here (sha256-checked); no Go, so govulncheck says
          # NOT CHECKED in CI, out loud. Warn-first either way: the scan lists, it never fails this job.
          FELHOM_SCAN_FETCH: "1"
+          # R-1327: test_image_runtime.py reads a zstd layer with the PINNED zstandard wheel (sha256-checked); with this
+          # set, a missing decoder FAILS the suite instead of saying NOT CHECKED.
+          FELHOM_WHEEL_FETCH: "1"
        run: python3 scripts/repo_gates.py --fast --scope="${PUSH_SCOPE:-code}"

      - name: Manual proof - alarm key present, optional forced failure (dispatch only, R-1129)
@@ -51,6 +51,9 @@ jobs:
          RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
          GH_ADVISORY_TOKEN: ${{ secrets.GH_ADVISORY_TOKEN }}
          MODE: ${{ github.event.inputs.mode }}
+          # R-1327: zstd layers (Home Assistant and others) are read with the PINNED zstandard wheel, sha256-checked
+          # (scripts/image_runtime.py); without it they are NOT CHECKED, out loud.
+          FELHOM_WHEEL_FETCH: "1"
        run: |
          set -u
          mkdir -p documentation/security-watch
@@ -691,9 +691,12 @@ needed the layers — Alpine/Debian package databases, Node's `node_version.h`,
 matches it against `scripts/security_watch_runtimes.json`; a hit is an ADVISORY with source „runtime", a read that
 fails is NOT CHECKED. Results are cached by the tag's digest in `state.json` (`runtime_cache`) and the nightly lookup is
 a HEAD request (Docker Hub does not count it against the anonymous pull limit — the first run's GETs hit HTTP 429).
-**Decided by CC unattended 2026-10-27 — operator may reverse:** the CI runner has no zstd (Python 3.13 has none), so
-images with zstd layers (Home Assistant) are read on DooPlex by a filling run (`--no-mail`, the stored Docker Hub login
-through `sec run`) and committed into the cache; CI reads them from there and lists a new unread zstd image as NOT
-CHECKED — instead of a new dependency in CI. The DooPlex group (`documentation/security-watch/dooplex-workloads.txt`,
+**The cache fills itself in CI (2026-10-28, R-1327; replaces the 2026-10-27 night's DooPlex filling run).** Images with
+zstd layers (Home Assistant) are read with the `zstandard` package, version 0.25.0, as a wheel pinned by the sha256 PyPI
+publishes and checked every time it is opened; the workflows fetch it (`FELHOM_WHEEL_FETCH=1`) into the runner's cache.
+Not the `zstd` program in the CI runner image: that image is built in the homelab repo, and changing it is a DooPlex
+change. A missing or wrong wheel makes the zstd layers NOT CHECKED, said in the log's `RUNTIME` line — never a pass. A
+new image is read on the night its tag first appears; the log's `RUNTIME read=… (… s) from-cache=…` line says how much
+was read and how long it took. The DooPlex group (`documentation/security-watch/dooplex-workloads.txt`,
 from homelab-manifests) is read the same way and mailed in its own block; Felhom changes nothing on DooPlex.

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""immich_repair.py EVDIR — re-prove Immich's after_unclean_boot scripts on v3.3.5 on scratch 9202 (the method of
+felhom.eu audits/immich-repair-2026-10-14 Part C): deploy through the product (9202 pointed at the drill, Immich
+v3.3.5), the admin through the setup gate, a family member, 20 + 10 photos through Immich's own API, then damage
+4 thumbnail files and delete 1 original, run the template's repair and report_lost commands inside immich-server
+exactly as the controller does (node -e <script> <ids…>), and check: 0 empty files after, 0 controls rewritten,
+the deleted original reported by name. Remove through the product at the end. No password is printed."""
+import hashlib, io, json, os, re, sys, tempfile, time
+
+sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
+import box_walk as w                      # noqa: E402
+import upgrade_fixtures_box28 as fx       # noqa: E402
+import yaml                               # noqa: E402
+from PIL import Image                     # noqa: E402
+
+EV = sys.argv[1]
+os.makedirs(EV, exist_ok=True)
+DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
+OUT = []
+
+
+def say(*a):
+    w.say(*a)
+    OUT.append(w.LOG[-1])
+
+
+def save():
+    open(os.path.join(EV, "repair-v3.3.5.txt"), "w").write("\n".join(OUT) + "\n")
+
+
+def jpeg(i):
+    im = Image.new("RGB", (640, 480), ((i * 37) % 256, (i * 91) % 256, (i * 53) % 256))
+    for y in range(0, 480, 16):
+        for x in range(0, 640, 16):
+            if (x // 16 + y // 16 + i) % 3 == 0:
+                im.paste(((x * i) % 256, (y * 7) % 256, (i * 13) % 256), (x, y, x + 16, y + 16))
+    b = io.BytesIO()
+    im.save(b, "JPEG", quality=85)
+    return b.getvalue()
+
+
+def login(sub, u, pw):
+    rc, code, out = w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json",
+                               data=json.dumps({"email": u, "password": pw}), method="POST")
+    return json.loads(out)["accessToken"]
+
+
+def upload(sub, tok, n, tag, tmp):
+    ok = 0
+    for i in range(n):
+        p = os.path.join(tmp, f"{tag}-{i:03d}.jpg")
+        open(p, "wb").write(jpeg(i + (100 if tag == "mem" else 0)))
+        ts = "2026-01-%02dT%02d:00:00.000Z" % (1 + i % 28, i % 24)
+        rc, code, out = w.app_curl(sub, "/api/assets", "-H", f"Authorization: Bearer {tok}",
+                                   "-F", f"assetData=@{p};type=image/jpeg", "-F", f"deviceAssetId={tag}-{i}",
+                                   "-F", "deviceId=felhom-drill", "-F", f"fileCreatedAt={ts}", "-F", f"fileModifiedAt={ts}",
+                                   method="POST", timeout=90)
+        if code in ("200", "201"):
+            ok += 1
+        else:
+            say(f"  upload {tag} {i} -> {code} {out[:150]}")
+    return ok
+
+
+def node_cmd(block):
+    cmd = block["command"]
+    assert cmd[:2] == ["node", "-e"], cmd
+    return cmd[2]
+
+
+def main():
+    fy = yaml.safe_load(open(f"{DRILL}/templates/immich/.felhom.yml"))
+    aub = fy["after_unclean_boot"]
+    comp = open(f"{DRILL}/templates/immich/docker-compose.yml").read()
+    img = re.search(r"image:\s*(ghcr.io/immich-app/immich-server:\S+)", comp).group(1)
+    say(f"drill head {os.popen(f'git -C {DRILL} rev-parse --short HEAD').read().strip()}: immich-server {img}; "
+        f"the block lists {aub['images']}")
+    w.login()
+    if w.stack("immich").get("deployed"):
+        say("immich already deployed on 9202 — STOP (not ours)")
+        return 2
+    if not w.deploy("immich", "photos"):
+        return 1
+    pins = (w.stack("immich").get("app_config") or {}).get("pinned_images") or {}
+    say(f"deployed: {pins.get('immich-server')}")
+    if pins.get("immich-server") != img:
+        say("the box did not deploy the drill's v3.3.5 — STOP")
+        return 1
+    seed = fx.Immich().seed(w, "photos", say)
+    if not seed:
+        say("admin sign-up failed")
+        return 1
+    atok = login("photos", seed["u"], seed["pw"])
+    mu, mpw = "member" + os.urandom(3).hex() + "@example.invalid", "Drill-" + os.urandom(8).hex()
+    rc, code, out = w.app_curl("photos", "/api/admin/users", "-H", f"Authorization: Bearer {atok}",
+                               "-H", "Content-Type: application/json",
+                               data=json.dumps({"email": mu, "password": mpw, "name": "Member"}), method="POST")
+    say(f"family member: {code}")
+    mtok = login("photos", mu, mpw)
+    tmp = tempfile.mkdtemp()
+    say(f"admin uploaded: {upload('photos', atok, 20, 'adm', tmp)}")
+    say(f"member uploaded: {upload('photos', mtok, 10, 'mem', tmp)}")
+    hdd = w.app_env("immich", "HDD_PATH")
+    under = aub["empty_files"]["under"].replace("${HDD_PATH}", hdd)
+    up = under.rsplit("/thumbs", 1)[0]
+    for _ in range(60):
+        n = w.guest(f"find {under} -type f -name '*_thumbnail.webp' -size +0 | wc -l; "
+                    f"find {under} -type f -name '*_preview.jpeg' -size +0 | wc -l").split()
+        if n == ["30", "30"]:
+            break
+        time.sleep(5)
+    say(f"thumbnails/previews settled: {n}")
+    srv = w.guest("docker ps --filter label=com.docker.compose.service=immich-server --format '{{.Names}}'").strip()
+    # controls: every thumbs file before the damage (size + mtime)
+    ctl0 = w.guest(f"find {under} -type f -printf '%p %s %T@\\n' | sort")
+    files = [l.split()[0] for l in ctl0.splitlines() if l.endswith(tuple("0123456789")) and "_thumbnail.webp" in l]
+    by_user = {}
+    for f in files:
+        by_user.setdefault(f.split("/thumbs/")[1].split("/")[0], []).append(f)
+    users = sorted(by_user)
+    victims = by_user[users[0]][:2] + by_user[users[1]][:2]
+    rx = re.compile(aub["empty_files"]["id_from_name"])
+    ids = [rx.search(os.path.basename(v)).group(1) for v in victims]
+    # one original deleted (not a victim's), as the power cut can lose a file that was being written
+    orig = w.guest(f"find {up}/upload {up}/library -type f -name '*.jpg' 2>/dev/null | head -1").strip()
+    w.guest("\n".join(f": > {v}" for v in victims) + f"\nrm -f {orig}\n")
+    empty0 = w.guest(f"find {under} -type f -size 0 | wc -l").strip()
+    say(f"damage: {empty0} empty files ({len(users)} users), 1 original deleted")
+    # the repair, exactly as the controller runs it: node -e <script> <ids…> inside the service's container
+    t = f"/tmp/felhom-aub-{os.getpid()}.js"
+    rep = node_cmd(aub["repair"])
+    out = w.guest(f"cat > {t} <<'FELHOMJS'\n{rep}\nFELHOMJS\nt0=$(date +%s%N)\n"
+                  f"docker exec {srv} node -e \"$(cat {t})\" {' '.join(ids)}; echo \"rc=$? in $(( ($(date +%s%N) - t0) / 1000000 )) ms\"\n"
+                  f"rm -f {t}", timeout=900)
+    say("== repair with %d ids\n%s" % (len(ids), out.strip()))
+    empty1 = w.guest(f"find {under} -type f -size 0 | wc -l").strip()
+    ctl1 = w.guest(f"find {under} -type f -printf '%p %s %T@\\n' | sort")
+    a = {l.split()[0]: l for l in ctl0.splitlines() if l.strip()}
+    b = {l.split()[0]: l for l in ctl1.splitlines() if l.strip()}
+    # a CONTROL is a file of an asset that was NOT damaged; the job rebuilds a damaged asset's thumbnail AND preview,
+    # so its sibling files are expected to change (the 2026-10-14 method counted the same way)
+    controls = [p for p in a if not any(os.path.basename(p).startswith(i) for i in ids) and not p.endswith("/.immich")]
+    rewritten = [p for p in controls if b.get(p) != a[p]]
+    sib = [os.path.basename(p) for p in a if p not in victims and any(os.path.basename(p).startswith(i) for i in ids)]
+    say(f"damaged assets' other files (rebuilt by the job, expected): {len(sib)}; changed: "
+        f"{sum(1 for p in a if os.path.basename(p) in sib and b.get(p) != a[p])}")
+    if rewritten:
+        say("controls rewritten: " + ", ".join(os.path.basename(p) for p in rewritten))
+    fixed = [v for v in victims if v in b and b[v].split()[1] != "0"]
+    say(f"after: empty now {empty1}; victims repaired {len(fixed)} of {len(victims)}; "
+        f"controls rewritten {len(rewritten)} of {len(controls)}")
+    lost = node_cmd(aub["report_lost"])
+    out = w.guest(f"cat > {t} <<'FELHOMJS'\n{lost}\nFELHOMJS\n"
+                  f"docker exec {srv} node -e \"$(cat {t})\"; echo \"rc=$?\"\nrm -f {t}", timeout=900)
+    say("== report_lost (the deleted original: %s)\n%s" % (os.path.basename(orig), out.strip()))
+    verdict = (empty1 == "0" and len(fixed) == len(victims) and not rewritten and "FELHOM_REPAIR_OK" in "\n".join(OUT)
+               and "FELHOM_LOST_OK lost=1" in out)
+    say(f"VERDICT {'proven' if verdict else 'NOT proven'} on {img}")
+    save()
+    return 0 if verdict else 1
+
+
+if __name__ == "__main__":
+    rc = 1
+    try:
+        rc = main()
+    finally:
+        try:
+            if w.stack("immich").get("deployed"):
+                w.remove("immich")
+                OUT.extend(w.LOG[-3:])
+        finally:
+            save()
+    sys.exit(rc)
@@ -0,0 +1,20 @@
+07:49:39 drill head 34b14d5: immich-server ghcr.io/immich-app/immich-server:v3.3.5; the block lists ['ghcr.io/immich-app/immich-server:v3.2.4']
+07:51:25 deployed: ghcr.io/immich-app/immich-server:v3.3.5
+07:51:25     immich: seeded album drillalbumf9eea650
+07:51:25 family member: 201
+07:51:27 admin uploaded: 20
+07:51:27 member uploaded: 10
+07:51:44 thumbnails/previews settled: ['30', '30']
+07:52:08 damage: 4 empty files (2 users), 1 original deleted
+07:52:14 == repair with 4 ids
+FELHOM_REPAIR_OK queued=4 waited_ms=2003
+rc=0 in 2348 ms
+07:52:21 after: empty now 0; victims repaired 4 of 4; controls rewritten 4 of 56
+07:52:27 == report_lost (the deleted original: e9bab540-26d7-46dd-9e92-5a7738ffb99e.jpg)
+LOST {"name":"adm-006.jpg","taken":"2026-01-07 07:00"}
+FELHOM_LOST_OK lost=1 waited_ms=2035
+rc=0
+07:52:27 VERDICT NOT proven on ghcr.io/immich-app/immich-server:v3.3.5
+07:52:29   [X] stop -> 200 {'ok': True, 'message': 'Stack immich stop completed'}
+07:53:01   [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'immich', 'volumes_removed': ['immich_immich_ml_cache', 'immich_immich_postgres_data'], 'hdd_paths_removed': ['/mnt/felhom-driv
+07:53:10   [X] after remove: deployed=False leftovers=''
@@ -0,0 +1,21 @@
+07:53:25 drill head 34b14d5: immich-server ghcr.io/immich-app/immich-server:v3.3.5; the block lists ['ghcr.io/immich-app/immich-server:v3.2.4']
+07:54:56 deployed: ghcr.io/immich-app/immich-server:v3.3.5
+07:54:57     immich: seeded album drillalbum5cd41f9e
+07:54:58 family member: 201
+07:54:59 admin uploaded: 20
+07:55:00 member uploaded: 10
+07:55:15 thumbnails/previews settled: ['30', '30']
+07:55:37 damage: 4 empty files (2 users), 1 original deleted
+07:55:43 == repair with 4 ids
+FELHOM_REPAIR_OK queued=4 waited_ms=2004
+rc=0 in 2592 ms
+07:55:52 damaged assets' other files (rebuilt by the job, expected): 4; changed: 4
+07:55:52 after: empty now 0; victims repaired 4 of 4; controls rewritten 0 of 52
+07:55:59 == report_lost (the deleted original: 5b2bafda-860a-422b-b980-6b34049494c0.jpg)
+LOST {"name":"adm-016.jpg","taken":"2026-01-17 17:00"}
+FELHOM_LOST_OK lost=1 waited_ms=3045
+rc=0
+07:55:59 VERDICT proven on ghcr.io/immich-app/immich-server:v3.3.5
+07:56:01   [X] stop -> 200 {'ok': True, 'message': 'Stack immich stop completed'}
+07:56:34   [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'immich', 'volumes_removed': ['immich_immich_ml_cache', 'immich_immich_postgres_data'], 'hdd_paths_removed': ['/mnt/felhom-driv
+07:56:42   [X] after remove: deployed=False leftovers=''
@@ -0,0 +1,15 @@
+mirror main 34b14d50
+
+26:    repo_url: file:///opt/docker/felhom-controller/drill-catalog.git
+
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated home-assistant/.felhom.yml
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated immich/docker-compose.yml
+2026/10/28 06:49:03 sync.go:535: [INFO] [sync] Updated immich/.felhom.yml
+2026/10/28 06:49:04 sync.go:261: [INFO] [sync] catalog moved b5b78f61cf7e -> 34b14d5003f5 (0 new, 2 stack dir(s) changed)
+2026/10/28 06:49:04 sync.go:299: [INFO] [sync] Catalog sync complete
+2026/10/28 06:49:04 sync.go:142: [INFO] [sync] Initial sync: Sablonok frissítve — frissítve: home-assistant, immich
+
+  expect n8n/docker-compose.yml ∋ N8N_DIAGNOSTICS_ENABLED=false: 1
+  expect opengist/docker-compose.yml ∋ 256M: 2
+  expect homepage/.felhom.yml ∋ family_gate: true: 1
+drill main 34b14d5, box cache 34b14d5: DRILL-CONFIRMED
@@ -0,0 +1,16 @@
+RESTORED-IDENTICAL
+26:    repo_url: https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated home-assistant/.felhom.yml
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated immich/docker-compose.yml
+2026/10/28 06:57:05 sync.go:535: [INFO] [sync] Updated immich/.felhom.yml
+2026/10/28 06:57:05 sync.go:261: [INFO] [sync] catalog moved 34b14d5003f5 -> b5b78f61cf7e (0 new, 2 stack dir(s) changed)
+2026/10/28 06:57:05 sync.go:299: [INFO] [sync] Catalog sync complete
+2026/10/28 06:57:05 sync.go:142: [INFO] [sync] Initial sync: Sablonok frissítve — frissítve: home-assistant, immich
+
+26:    repo_url: https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+-rw------- 1 root root 1994 Oct 22 14:47 /var/lib/docker/volumes/felhom-controller-data/_data/controller.yaml.pre-release1026-drill
+mirror main 34b14d50
+cache HEAD b5b78f6
+https://gitea.dooplex.hu/admin/app-catalog-felhom.eu.git
+
@@ -2565,19 +2565,6 @@
   "python": null,
   "unchecked": []
  },
-  "sha256:3e6710a7ab2a61311d9d899b719f6c3657791c63e8f4942cec4ebc42401d6b76": {
-   "apk": {
-    "brotli-libs": "1.2.0-r1"
-   },
-   "brotli": "1.2.0",
-   "brotlicffi": null,
-   "digest": "sha256:e47c978e1b801466e7f62f612fd552bc3a228e077b31a3f1c22c05cf63d754da",
-   "dpkg": {},
-   "env": {},
-   "node": null,
-   "python": null,
-   "unchecked": []
-  },
  "sha256:40c454d74ffdd9ecb15833e4d96f3a630d59aad862a1869381ae6ee086454419": {
   "apk": {},
   "brotli": null,
@@ -3107,20 +3094,6 @@
   "python": null,
   "unchecked": []
  },
-  "sha256:8677fb7b81b1d3db20b849e989a1dadc2a11159e51eb34a2acdcb01102274edf": {
-   "apk": {
-    "brotli-libs": "1.2.1-r0",
-    "nodejs": "24.18.1-r0"
-   },
-   "brotli": null,
-   "brotlicffi": null,
-   "digest": "sha256:923a808be529a834a171b3e045821aeb7e2851f4b6fc9514076785b68a9142f6",
-   "dpkg": {},
-   "env": {},
-   "node": "24.18.1",
-   "python": null,
-   "unchecked": []
-  },
  "sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef": {
   "apk": {
    "brotli-libs": "1.2.0-r1"
@@ -1,3 +1,16 @@
+## 2026-10-28 — the security watch reads zstd layers in CI by itself (R-1327)
+
+- **`image_runtime.py`:** zstd layers are decoded by `zstandard` 0.25.0 — an importable one, else the wheel pinned by
+  PyPI's sha256 for this Python and libc (`ZSTANDARD_WHEELS`: CPython 3.12/3.13, musl/glibc), from
+  `~/.cache/felhom-wheels`, fetched only with `FELHOM_WHEEL_FETCH=1` (both workflows set it) — else the `zstd` program,
+  else a NOTE (NOT CHECKED). The wheel is hash-checked every time it is opened and loaded from the checked bytes. Any
+  error while a layer decodes is a NOTE, never a pass.
+- **`security_watch.py`:** a `RUNTIME read=N (S s) from-cache=M not-checked=K zstd-decoder=…` line and one
+  `RUNTIME-READ <image> <s> ok|NOT CHECKED` line per image read tonight.
+- Tests: `test_image_runtime.py` 22 checks (zstd above gzip with no `zstd` program, a damaged and a cut-off zstd layer,
+  no decoder, the wheel loader's hash refusals) — the zstd read seen failing on the 2026-10-27 module first;
+  `test_security_watch.py` W10 counts (59 checks).
+
 ## 2026-10-27 — the security watch reads two notice shapes right (R-1310, part)

 - **`security_watch.py` `ghsa_affects`:** a GitHub Actions advisory (ecosystem `actions`, the project's own CI) is no
@@ -12,18 +12,30 @@ boxes ran the bad code. This module reads what the image itself carries:
     `brotli-*` / `brotlicffi-*` `.dist-info` folders of Python. The topmost layer that holds a file wins (a later layer
     replaces an earlier one's copy).

-What it cannot read is said, never guessed: a registry it has no token route for, a zstd layer where no `zstd` program exists (Python 3.13 has no zstd),
-a layer over the byte budget — each makes the result `unchecked` with its reason, and the watch prints NOT CHECKED.
+ZSTD LAYERS (R-1327, 2026-10-28). Home Assistant and other images ship zstd-compressed layers, and Python before 3.14
+has no zstd. The decoder is the `zstandard` package (0.25.0), pinned by the sha256 PyPI publishes for each wheel: an
+importable `zstandard` first, else the pinned wheel from the cache (`~/.cache/felhom-wheels`; fetched only when
+FELHOM_WHEEL_FETCH=1 — the CI workflows set it), else the `zstd` program. Not the CI runner image: that image is built
+in the homelab repo and a change there is a DooPlex change. The wheel's hash is checked every time it is opened.
+
+What it cannot read is said, never guessed: a registry it has no token route for, a zstd layer where no decoder exists,
+a layer that does not decode, a layer over the byte budget — each makes the result `unchecked` with its reason, and the
+watch prints NOT CHECKED.

 USAGE (also a library: `read_image(ref)` -> dict)
  python3 scripts/image_runtime.py IMAGE[:TAG] ... [--budget-mb=N] [--json]
 """
+import atexit
+import hashlib
 import json
+import os
+import platform
 import re
 import shutil
 import subprocess
 import sys
-import zlib
+import tempfile
+import zipfile
 import tarfile
 import urllib.error
 import urllib.request
@@ -43,6 +55,101 @@ class Unreadable(Exception):
    pass


+ZSTANDARD_VERSION = "0.25.0"
+# (Python, libc) -> (wheel URL, sha256) — the digests PyPI publishes for the 0.25.0 files (released 2025-09-14), read
+# 2026-10-28. The CI runner is Alpine (musl) with Python 3.12; DooPlex is Debian (glibc) with Python 3.13.
+_PYPI = "https://files.pythonhosted.org/packages/"
+ZSTANDARD_WHEELS = {
+    ("3.12", "musl"): (_PYPI + "fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/"
+                       "zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl",
+                       "474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa"),
+    ("3.13", "musl"): (_PYPI + "b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/"
+                       "zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl",
+                       "4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf"),
+    ("3.12", "glibc"): (_PYPI + "53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/"
+                        "zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                        "5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902"),
+    ("3.13", "glibc"): (_PYPI + "63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/"
+                        "zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                        "8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a"),
+}
+_ZSTANDARD = {}   # the loaded module, or the reason there is none — decided once per process
+
+
+def _libc():
+    if platform.libc_ver()[0] == "glibc":
+        return "glibc"
+    import glob
+    return "musl" if glob.glob("/lib/ld-musl-*") else "unknown"
+
+
+def _wheel_cache():
+    return os.path.join(os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"),
+                        "felhom-wheels")
+
+
+def zstandard_module(fetch=None, wheels=None, cache=None, try_import=True):
+    """(module, how) or (None, why). An importable `zstandard` first; else the pinned wheel for this Python and libc,
+    from the cache or — only when `fetch` (default: FELHOM_WHEEL_FETCH=1) — from PyPI; its sha256 is checked before
+    every use, and a wheel with any other hash is never opened. Pinned by test_image_runtime.py (wheel_*)."""
+    if try_import:
+        try:
+            import zstandard
+            return zstandard, "zstandard %s (installed)" % getattr(zstandard, "__version__", "?")
+        except ImportError:
+            pass
+    fetch = os.environ.get("FELHOM_WHEEL_FETCH") == "1" if fetch is None else fetch
+    key = ("%d.%d" % sys.version_info[:2], _libc())
+    pin = (wheels if wheels is not None else ZSTANDARD_WHEELS).get(key)
+    if not pin:
+        return None, "no zstd decoder: no pinned zstandard wheel for Python %s / %s" % key
+    url, want = pin
+    cache = cache or _wheel_cache()
+    path = os.path.join(cache, url.rsplit("/", 1)[1])
+    data = open(path, "rb").read() if os.path.isfile(path) else None
+    if data is None or hashlib.sha256(data).hexdigest() != want:
+        if not fetch:
+            return None, "no zstd decoder: the pinned zstandard wheel is not in %s (FELHOM_WHEEL_FETCH=1 fetches it)" % cache
+        try:
+            data = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=60).read()
+        except Exception as e:  # noqa: BLE001 — network, DNS, TLS: the zstd layers are NOT CHECKED, said out loud
+            return None, "no zstd decoder: the pinned zstandard wheel could not be fetched (%s)" % str(e)[:100]
+        if hashlib.sha256(data).hexdigest() != want:
+            return None, "no zstd decoder: the fetched zstandard wheel's sha256 is not the pinned one — not used"
+        os.makedirs(cache, exist_ok=True)
+        with open(path + ".new", "wb") as f:
+            f.write(data)
+        os.replace(path + ".new", path)
+    into = tempfile.mkdtemp(prefix="felhom-zstandard-")
+    atexit.register(shutil.rmtree, into, True)
+    import io
+    with zipfile.ZipFile(io.BytesIO(data)) as z:   # from the bytes just hashed, never from the file again
+        z.extractall(into)
+    sys.path.insert(0, into)
+    try:
+        import importlib
+        sys.modules.pop("zstandard", None)
+        mod = importlib.import_module("zstandard")
+    except Exception as e:  # noqa: BLE001
+        return None, "no zstd decoder: the pinned zstandard wheel did not load (%s)" % str(e)[:100]
+    finally:
+        sys.path.remove(into)
+    return mod, "zstandard %s (pinned wheel, sha256 checked)" % getattr(mod, "__version__", "?")
+
+
+def zstd_decoder():
+    """How this process decodes a zstd layer: ('module', mod) | ('program', 'zstd') | (None, why). Decided once."""
+    if "d" not in _ZSTANDARD:
+        mod, how = zstandard_module()
+        if mod:
+            _ZSTANDARD["d"] = ("module", mod)
+        elif shutil.which("zstd"):
+            _ZSTANDARD["d"] = ("program", "zstd")
+        else:
+            _ZSTANDARD["d"] = (None, how)
+    return _ZSTANDARD["d"]
+
+
 def split_ref(ref):
    """'rommapp/romm:5.4.1' -> (registry, path, tag). lscr.io is Docker Hub's mirror (the watch's rule)."""
    tag = "latest"
@@ -152,15 +259,18 @@ def parse_node_h(text):
    return ".".join(p.group(1) for p in parts) if all(parts) else None


-def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
-    """Stream layers top-down. Returns (found, notes): found holds apk/dpkg/node_h/pydist; the first (topmost) wins."""
+def scan_layers(reg, layers, budget=DEFAULT_BUDGET, decoder=None):
+    """Stream layers top-down. Returns (found, notes): found holds apk/dpkg/node_h/pydist; the first (topmost) wins.
+    `decoder` is zstd_decoder()'s answer (a seam for the tests); a zstd layer with none is a NOTE, never skipped quietly."""
    found, notes, spent = {}, [], 0
    for layer in reversed(layers):
        mt, size = layer.get("mediaType", ""), int(layer.get("size") or 0)
        zstd = "zstd" in mt
-        if zstd and not shutil.which("zstd"):
-            notes.append("zstd layer %s: no zstd program here" % layer["digest"][:19])
-            continue
+        if zstd:
+            kind, dec = decoder or zstd_decoder()
+            if not kind:
+                notes.append("zstd layer %s: %s" % (layer["digest"][:19], dec))
+                continue
        if spent + size > budget:
            notes.append("byte budget reached before layer %s (%d MB)" % (layer["digest"][:19], size >> 20))
            break
@@ -168,7 +278,9 @@ def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
        resp = reg.blob(layer["digest"], stream=True)
        proc = None
        try:
-            if zstd:   # Python 3.13 has no zstd: the zstd program decompresses the stream (a pipe, nothing on disk)
+            if zstd and kind == "module":   # the pinned zstandard: a streaming reader, nothing on disk
+                fileobj, mode = dec.ZstdDecompressor().stream_reader(resp, read_across_frames=True), "r|"
+            elif zstd:   # the zstd program decompresses the stream (a pipe, nothing on disk)
                proc = subprocess.Popen(["zstd", "-dc"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
                import threading

@@ -202,7 +314,7 @@ def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
                        if pm:
                            key = pm.group(1).lower()
                            found.setdefault("pydist", {}).setdefault(key, pm.group(2))
-        except (tarfile.TarError, OSError, EOFError, zlib.error) as e:
+        except Exception as e:  # noqa: BLE001 — tar, zlib, zstd (ZstdError), I/O: the layer is NOT CHECKED, never a pass
            notes.append("layer %s unreadable: %s" % (layer["digest"][:19], type(e).__name__))
        finally:
            if proc:
@@ -275,6 +275,14 @@ def cached_runtime_reader(cache, resolve=None, read=None):
    caches a COMPLETE result only (an unchecked one is tried again the next night). `cache` lives in the state file."""
    resolve = resolve or image_runtime.resolve_digest
    read = read or image_runtime.read_image
+    import threading
+    import time
+    lock = threading.Lock()
+    stats = {"cached": 0, "read": 0, "read_s": 0.0, "not_checked": 0}
+
+    def count(key, n=1):
+        with lock:
+            stats[key] += n

    def reader(ref):
        try:
@@ -287,12 +295,21 @@ def cached_runtime_reader(cache, resolve=None, read=None):
            hit = dict(cache[digest], ref=ref)
            hit["_used"] = True
            cache[digest]["_seen"] = True
+            count("cached")
            return hit
+        t0 = time.monotonic()
        res = read(ref)
+        count("read_s", time.monotonic() - t0)
+        count("read")
+        print("RUNTIME-READ %s %.0f s %s" % (ref, time.monotonic() - t0, "NOT CHECKED: " + "; ".join(res["unchecked"])[:160]
+                                             if res.get("unchecked") else "ok"), flush=True)
        if digest and not res.get("unchecked"):
            cache[digest] = {k: v for k, v in res.items() if k != "ref"}
            cache[digest]["_seen"] = True
+        else:
+            count("not_checked")
        return res
+    reader.stats = stats   # the RUNTIME line of the log: how much was read tonight and how long it took (R-1327)
    return reader


@@ -855,14 +872,19 @@ def main(argv):
    if state_path and os.path.isfile(state_path):
        prior = json.load(open(state_path, encoding="utf-8"))
    rcache = {k: dict(v, _seen=False) for k, v in (prior.get("runtime_cache") or {}).items()}
-    run(items, net, today, rotate=not token,
-        runtime=None if "no-runtime" in opts else cached_runtime_reader(rcache))
+    rreader = None if "no-runtime" in opts else cached_runtime_reader(rcache)
+    run(items, net, today, rotate=not token, runtime=rreader)
    for line in table(items):
        print(line)
    counts = {}
    for it in items:
        counts[it["class"]] = counts.get(it["class"], 0) + 1
    print("COUNTS " + " ".join("%s=%d" % (k.replace(" ", "_"), v) for k, v in sorted(counts.items())))
+    if rreader:
+        st = rreader.stats
+        print("RUNTIME read=%d (%.0f s) from-cache=%d not-checked=%d zstd-decoder=%s" % (
+            st["read"], st["read_s"], st["cached"], st["not_checked"],
+            image_runtime.zstd_decoder()[1] if st["read"] else "not needed"))
    print("REQUESTS " + " ".join("%s=%d" % kv for kv in sorted(net.requests.items())))
    state_path = opts.get("state")
    state, missing = {}, True
@@ -6,12 +6,22 @@ version header gives the version; an image reference splits the way the watch's
 Hub; a bare name is library/); a tar layer streamed top-down returns the TOPMOST copy of a file and the Python
 `.dist-info` versions; the DooPlex list keeps one line per image:tag with every file that names it, and skips templated
 and variable images. Red-proof: a scan that keeps the FIRST (bottom) copy instead of the topmost fails `topmost`.
+
+ZSTD (R-1327, 2026-10-28): a zstd layer above a gzip one is read with NO `zstd` program on the PATH (the CI runner has
+none) — through the pinned `zstandard` wheel; a zstd layer that does not decode, and one with no decoder at all, are
+NOTES (the watch's NOT CHECKED), never a pass; the wheel loader opens a wheel only when its sha256 is the pinned one.
+Where no decoder can be had (no wheel in the cache, FELHOM_WHEEL_FETCH unset) the read check says NOT CHECKED out loud
+and is not counted; with FELHOM_WHEEL_FETCH=1 (CI sets it) a missing decoder FAILS. Red-proof: the 2026-10-27 module
+(the `zstd` program or nothing) fails `zstd: a zstd layer is read with no zstd program`.
 """
+import base64
+import hashlib
 import io
 import os
 import sys
 import tarfile
 import tempfile
+import zipfile

 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
@@ -47,6 +57,78 @@ class FakeReg:
        return io.BytesIO(self.blobs[digest])


+# A tar holding an Alpine package database (nodejs 24.22.1, brotli-libs 1.2.1), compressed by `zstd -19` (2026-10-28).
+ZSTD_LAYER = base64.b64decode(
+    "KLUv/QRoLQMAsgUSEJB9hC/xJSHf5yhLWm0eYqY+iotURKotdmoB8FiUYcw4RjGQJXy2quX/jpvO2z94QaiU9/dP/+/npPcHKW0610jcI/M24iI"
+    "NCiAwMT3M5Vx0jvfWN08dUPKASwQMGmAZDXf7YaaA")
+ZSTD_MT = "application/vnd.oci.image.layer.v1.tar+zstd"
+GZ_MT = "application/vnd.oci.image.layer.v1.tar+gzip"
+
+
+def zstd_checks():
+    gz = tar_bytes({"usr/local/lib/python3.12/site-packages/brotli-1.2.1.dist-info/RECORD": b"x",
+                    "lib/apk/db/installed": b"P:nodejs\nV:20.0.0-r0\n"})
+    layers = [{"digest": "sha256:gz", "mediaType": GZ_MT, "size": len(gz)},
+              {"digest": "sha256:zst", "mediaType": ZSTD_MT, "size": len(ZSTD_LAYER)}]
+    reg = FakeReg({"sha256:gz": gz, "sha256:zst": ZSTD_LAYER})
+    path = os.environ.get("PATH", "")
+    os.environ["PATH"] = tempfile.mkdtemp()          # no `zstd` program: the CI runner's case
+    ir._ZSTANDARD.clear()                            # the decoder is decided once per process: decide it HERE
+    try:
+        found, notes = ir.scan_layers(reg, layers)
+    finally:
+        os.environ["PATH"] = path
+        ir._ZSTANDARD.clear()
+    no_decoder = [n for n in notes if "no zstd decoder" in n]
+    if no_decoder and os.environ.get("FELHOM_WHEEL_FETCH") != "1":
+        print("  NOT CHECKED  zstd: a zstd layer is read with no zstd program — %s" % no_decoder[0])
+    else:
+        check("zstd: a zstd layer is read with no zstd program (the top layer's apk db wins)",
+              (found.get("apk") or {}).get("nodejs") == "24.22.1-r0" and not notes, (found.get("apk"), notes))
+        check("zstd: the gzip layer under it is still read", (found.get("pydist") or {}).get("brotli") == "1.2.1",
+              found.get("pydist"))
+    found, notes = ir.scan_layers(reg, [{"digest": "sha256:zst", "mediaType": ZSTD_MT, "size": 9}],
+                                  decoder=(None, "no zstd decoder: test"))
+    check("zstd: no decoder is a NOTE (NOT CHECKED), never a silent pass", notes and "no zstd decoder" in notes[0]
+          and not found, (found, notes))
+    mod, _how = ir.zstandard_module()
+    if mod:
+        bad = ZSTD_LAYER[:40] + b"\x00" * 30 + ZSTD_LAYER[70:]
+        found, notes = ir.scan_layers(FakeReg({"sha256:bad": bad}), [{"digest": "sha256:bad", "mediaType": ZSTD_MT,
+                                                                      "size": len(bad)}], decoder=("module", mod))
+        check("zstd: a layer that does not decode is a NOTE (NOT CHECKED), never a pass",
+              any("unreadable" in n for n in notes), (found, notes))
+        found, notes = ir.scan_layers(FakeReg({"sha256:t": ZSTD_LAYER[:60]}), [{"digest": "sha256:t", "mediaType": ZSTD_MT,
+                                                                                "size": 60}], decoder=("module", mod))
+        check("zstd: a cut-off layer is a NOTE (NOT CHECKED), never a pass", any("unreadable" in n for n in notes),
+              (found, notes))
+
+
+def wheel_checks():
+    """The loader opens a wheel only when its sha256 is the pinned one. A fake `zstandard` wheel, served by file://."""
+    d = tempfile.mkdtemp()
+    whl = os.path.join(d, "zstandard-0.0-fake.whl")
+    with zipfile.ZipFile(whl, "w") as z:
+        z.writestr("zstandard/__init__.py", "__version__ = 'fake-for-test'\n")
+    good = hashlib.sha256(open(whl, "rb").read()).hexdigest()
+    key = ("%d.%d" % sys.version_info[:2], ir._libc())
+    url = "file://" + whl
+    mod, how = ir.zstandard_module(fetch=True, wheels={key: (url, good)}, cache=os.path.join(d, "c1"), try_import=False)
+    check("wheel: the pinned wheel is fetched, hash-checked and loaded", mod is not None and
+          getattr(mod, "__version__", "") == "fake-for-test" and "sha256 checked" in how, how)
+    mod, how = ir.zstandard_module(fetch=True, wheels={key: (url, "0" * 64)}, cache=os.path.join(d, "c2"), try_import=False)
+    check("wheel: a wheel whose sha256 is not the pinned one is never loaded", mod is None and "not the pinned" in how, how)
+    c3 = os.path.join(d, "c3")
+    os.makedirs(c3)
+    open(os.path.join(c3, os.path.basename(whl)), "wb").write(b"changed")
+    mod, how = ir.zstandard_module(fetch=False, wheels={key: (url, good)}, cache=c3, try_import=False)
+    check("wheel: a changed wheel in the cache is not loaded, and without FETCH nothing is fetched", mod is None and
+          "not in" in how, how)
+    mod, how = ir.zstandard_module(fetch=True, wheels={}, cache=c3, try_import=False)
+    check("wheel: no pin for this Python is a reason, never a guess", mod is None and "no pinned" in how, how)
+    sys.modules.pop("zstandard", None)
+
+
 def main():
    apk = ir.parse_apk("P:musl\nV:1.2.5-r10\n\nP:nodejs\nV:24.18.1-r0\n\nP:brotli-libs\nV:1.2.0-r1\n")
    check("apk database -> nodejs and brotli-libs", apk.get("nodejs") == "24.18.1-r0" and apk.get("brotli-libs") == "1.2.0-r1", apk)
@@ -75,6 +157,8 @@ def main():
    found, notes = ir.scan_layers(FakeReg({}), [{"digest": "sha256:z", "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd",
                                                "size": 10}], budget=0)
    check("a layer over the byte budget is a NOTE, never a silent pass", notes and ("budget" in notes[0] or "zstd" in notes[0]), notes)
+    zstd_checks()
+    wheel_checks()
    d = tempfile.mkdtemp()
    open(os.path.join(d, "a.yaml"), "w").write("spec:\n  containers:\n  - image: rommapp/romm:5.4.0\n    name: x\n"
                                                "  - image: '{{ .Values.image }}'\n  - image: $IMG\n")
@@ -389,7 +389,11 @@ def t_runtime():
    check("W10 an UNCHECKED result is not cached (tomorrow tries again)", "sha256:f" not in cache, list(cache))
    src = open(os.path.join(HERE, "security_watch.py"), encoding="utf-8").read()
    check("W10 main() hands the reader to run() (off only with --no-runtime)",
-          "runtime=None if \"no-runtime\" in opts else cached_runtime_reader(rcache))" in src)
+          "rreader = None if \"no-runtime\" in opts else cached_runtime_reader(rcache)" in src
+          and "runtime=rreader)" in src)
+    check("W10 the reader counts what it read and from the cache (the log's RUNTIME line)",
+          r.stats["read"] == 2 and r.stats["cached"] == 1 and r.stats["not_checked"] == 1 and "RUNTIME read=" in src,
+          r.stats)


 def t_ghcr_pages():