security watch reads zstd layers in CI by itself (R-1327): pinned zstandard 0.25.0 wheel, sha256-checked; RUNTIME log line; HA 2026.9.4 and RomM 5.4.1 dropped from the cache for the CI fill proof; Immich v3.3.5 power-cut repair evidence

scripts/image_runtime.py

@@ -12,18 +12,30 @@ boxes ran the bad code. This module reads what the image itself carries:
      `brotli-*` / `brotlicffi-*` `.dist-info` folders of Python. The topmost layer that holds a file wins (a later layer
      replaces an earlier one's copy).
 
-What it cannot read is said, never guessed: a registry it has no token route for, a zstd layer where no `zstd` program exists (Python 3.13 has no zstd),
-a layer over the byte budget — each makes the result `unchecked` with its reason, and the watch prints NOT CHECKED.
+ZSTD LAYERS (R-1327, 2026-10-28). Home Assistant and other images ship zstd-compressed layers, and Python before 3.14
+has no zstd. The decoder is the `zstandard` package (0.25.0), pinned by the sha256 PyPI publishes for each wheel: an
+importable `zstandard` first, else the pinned wheel from the cache (`~/.cache/felhom-wheels`; fetched only when
+FELHOM_WHEEL_FETCH=1 — the CI workflows set it), else the `zstd` program. Not the CI runner image: that image is built
+in the homelab repo and a change there is a DooPlex change. The wheel's hash is checked every time it is opened.
+
+What it cannot read is said, never guessed: a registry it has no token route for, a zstd layer where no decoder exists,
+a layer that does not decode, a layer over the byte budget — each makes the result `unchecked` with its reason, and the
+watch prints NOT CHECKED.
 
 USAGE (also a library: `read_image(ref)` -> dict)
   python3 scripts/image_runtime.py IMAGE[:TAG] ... [--budget-mb=N] [--json]
 """
+import atexit
+import hashlib
 import json
+import os
+import platform
 import re
 import shutil
 import subprocess
 import sys
-import zlib
+import tempfile
+import zipfile
 import tarfile
 import urllib.error
 import urllib.request
@@ -43,6 +55,101 @@ class Unreadable(Exception):
     pass
 
 
+ZSTANDARD_VERSION = "0.25.0"
+# (Python, libc) -> (wheel URL, sha256) — the digests PyPI publishes for the 0.25.0 files (released 2025-09-14), read
+# 2026-10-28. The CI runner is Alpine (musl) with Python 3.12; DooPlex is Debian (glibc) with Python 3.13.
+_PYPI = "https://files.pythonhosted.org/packages/"
+ZSTANDARD_WHEELS = {
+    ("3.12", "musl"): (_PYPI + "fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/"
+                       "zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl",
+                       "474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa"),
+    ("3.13", "musl"): (_PYPI + "b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/"
+                       "zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl",
+                       "4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf"),
+    ("3.12", "glibc"): (_PYPI + "53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/"
+                        "zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                        "5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902"),
+    ("3.13", "glibc"): (_PYPI + "63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/"
+                        "zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                        "8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a"),
+}
+_ZSTANDARD = {}   # the loaded module, or the reason there is none — decided once per process
+
+
+def _libc():
+    if platform.libc_ver()[0] == "glibc":
+        return "glibc"
+    import glob
+    return "musl" if glob.glob("/lib/ld-musl-*") else "unknown"
+
+
+def _wheel_cache():
+    return os.path.join(os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"),
+                        "felhom-wheels")
+
+
+def zstandard_module(fetch=None, wheels=None, cache=None, try_import=True):
+    """(module, how) or (None, why). An importable `zstandard` first; else the pinned wheel for this Python and libc,
+    from the cache or — only when `fetch` (default: FELHOM_WHEEL_FETCH=1) — from PyPI; its sha256 is checked before
+    every use, and a wheel with any other hash is never opened. Pinned by test_image_runtime.py (wheel_*)."""
+    if try_import:
+        try:
+            import zstandard
+            return zstandard, "zstandard %s (installed)" % getattr(zstandard, "__version__", "?")
+        except ImportError:
+            pass
+    fetch = os.environ.get("FELHOM_WHEEL_FETCH") == "1" if fetch is None else fetch
+    key = ("%d.%d" % sys.version_info[:2], _libc())
+    pin = (wheels if wheels is not None else ZSTANDARD_WHEELS).get(key)
+    if not pin:
+        return None, "no zstd decoder: no pinned zstandard wheel for Python %s / %s" % key
+    url, want = pin
+    cache = cache or _wheel_cache()
+    path = os.path.join(cache, url.rsplit("/", 1)[1])
+    data = open(path, "rb").read() if os.path.isfile(path) else None
+    if data is None or hashlib.sha256(data).hexdigest() != want:
+        if not fetch:
+            return None, "no zstd decoder: the pinned zstandard wheel is not in %s (FELHOM_WHEEL_FETCH=1 fetches it)" % cache
+        try:
+            data = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=60).read()
+        except Exception as e:  # noqa: BLE001 — network, DNS, TLS: the zstd layers are NOT CHECKED, said out loud
+            return None, "no zstd decoder: the pinned zstandard wheel could not be fetched (%s)" % str(e)[:100]
+        if hashlib.sha256(data).hexdigest() != want:
+            return None, "no zstd decoder: the fetched zstandard wheel's sha256 is not the pinned one — not used"
+        os.makedirs(cache, exist_ok=True)
+        with open(path + ".new", "wb") as f:
+            f.write(data)
+        os.replace(path + ".new", path)
+    into = tempfile.mkdtemp(prefix="felhom-zstandard-")
+    atexit.register(shutil.rmtree, into, True)
+    import io
+    with zipfile.ZipFile(io.BytesIO(data)) as z:   # from the bytes just hashed, never from the file again
+        z.extractall(into)
+    sys.path.insert(0, into)
+    try:
+        import importlib
+        sys.modules.pop("zstandard", None)
+        mod = importlib.import_module("zstandard")
+    except Exception as e:  # noqa: BLE001
+        return None, "no zstd decoder: the pinned zstandard wheel did not load (%s)" % str(e)[:100]
+    finally:
+        sys.path.remove(into)
+    return mod, "zstandard %s (pinned wheel, sha256 checked)" % getattr(mod, "__version__", "?")
+
+
+def zstd_decoder():
+    """How this process decodes a zstd layer: ('module', mod) | ('program', 'zstd') | (None, why). Decided once."""
+    if "d" not in _ZSTANDARD:
+        mod, how = zstandard_module()
+        if mod:
+            _ZSTANDARD["d"] = ("module", mod)
+        elif shutil.which("zstd"):
+            _ZSTANDARD["d"] = ("program", "zstd")
+        else:
+            _ZSTANDARD["d"] = (None, how)
+    return _ZSTANDARD["d"]
+
+
 def split_ref(ref):
     """'rommapp/romm:5.4.1' -> (registry, path, tag). lscr.io is Docker Hub's mirror (the watch's rule)."""
     tag = "latest"
@@ -152,15 +259,18 @@ def parse_node_h(text):
     return ".".join(p.group(1) for p in parts) if all(parts) else None
 
 
-def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
-    """Stream layers top-down. Returns (found, notes): found holds apk/dpkg/node_h/pydist; the first (topmost) wins."""
+def scan_layers(reg, layers, budget=DEFAULT_BUDGET, decoder=None):
+    """Stream layers top-down. Returns (found, notes): found holds apk/dpkg/node_h/pydist; the first (topmost) wins.
+    `decoder` is zstd_decoder()'s answer (a seam for the tests); a zstd layer with none is a NOTE, never skipped quietly."""
     found, notes, spent = {}, [], 0
     for layer in reversed(layers):
         mt, size = layer.get("mediaType", ""), int(layer.get("size") or 0)
         zstd = "zstd" in mt
-        if zstd and not shutil.which("zstd"):
-            notes.append("zstd layer %s: no zstd program here" % layer["digest"][:19])
-            continue
+        if zstd:
+            kind, dec = decoder or zstd_decoder()
+            if not kind:
+                notes.append("zstd layer %s: %s" % (layer["digest"][:19], dec))
+                continue
         if spent + size > budget:
             notes.append("byte budget reached before layer %s (%d MB)" % (layer["digest"][:19], size >> 20))
             break
@@ -168,7 +278,9 @@ def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
         resp = reg.blob(layer["digest"], stream=True)
         proc = None
         try:
-            if zstd:   # Python 3.13 has no zstd: the zstd program decompresses the stream (a pipe, nothing on disk)
+            if zstd and kind == "module":   # the pinned zstandard: a streaming reader, nothing on disk
+                fileobj, mode = dec.ZstdDecompressor().stream_reader(resp, read_across_frames=True), "r|"
+            elif zstd:   # the zstd program decompresses the stream (a pipe, nothing on disk)
                 proc = subprocess.Popen(["zstd", "-dc"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
                 import threading
 
@@ -202,7 +314,7 @@ def scan_layers(reg, layers, budget=DEFAULT_BUDGET):
                         if pm:
                             key = pm.group(1).lower()
                             found.setdefault("pydist", {}).setdefault(key, pm.group(2))
-        except (tarfile.TarError, OSError, EOFError, zlib.error) as e:
+        except Exception as e:  # noqa: BLE001 — tar, zlib, zstd (ZstdError), I/O: the layer is NOT CHECKED, never a pass
             notes.append("layer %s unreadable: %s" % (layer["digest"][:19], type(e).__name__))
         finally:
             if proc: