security watch: seven pins it can never read are a stated GAP, so the source streaks reset; a read timeout is retried; every failed source is printed (R-1334); R-1310's rules (another package, every GHSA a release names, a word-only line, date tags)

scripts/security_watch.py

@@ -7,7 +7,8 @@ publishes its security notices, and sorts every pin into exactly one class:
 
   ADVISORY      a notice affects the version we pin (with the fixed version when the notice gives one)
   NOT CHECKED   a source this pin needs failed tonight — printed and counted, never a silent pass
-  GAP           the pin has no machine-readable source (a stated `gap:` line); BEHIND is still shown
+  GAP           the pin has no machine-readable source (a stated `gap:` line), or the watch can never read it (our own
+                private registry; STATED_UNREADABLE) — listed with its reason in every weekly mail; BEHIND is still shown
   FLOATING      the tag names no exact version (`redis:7-alpine`), so no range can be matched; BEHIND still shown
   BEHIND        a newer version exists in the same channel, and no notice
   CURRENT       none of the above
@@ -79,6 +80,10 @@ VULNNOTES = {
 }
 STRONG = re.compile(r"\bCVE-\d{4}-\d{3,}|\bGHSA(?:-[0-9a-z]{4}){3}\b|vulnerab|security (?:fix|issue|update|release|"
                     r"advisory|patch|vulnerability)|\[security\]", re.I)
+GHSA = re.compile(r"\bGHSA(?:-[0-9a-z]{4}){3}\b", re.I)
+# A line where „security" is only a word (R-1310, 2026-11-02): Traefik v3.7.17's „Point security announcements to GitHub
+# advisories", Dawarich's „no longer receives security fixes". Pinned: test W16. A new shape is listed, the safe direction.
+WORD_ONLY = re.compile(r"security announcements?|security (?:policy|contact|researchers?)|no longer (?:receives?|gets?) security", re.I)
 FEED_WORDS = re.compile(r"vulnerab|compromis|\bCVE-\d{4}|security (?:update|fix|advisory|issue|notice)|exploit", re.I)
 
 
@@ -86,6 +91,10 @@ class SourceError(Exception):
     pass
 
 
+class Transient(SourceError):
+    """A transport failure (a read timeout, a reset): worth one more try. An HTTP answer is not transient, except a 429."""
+
+
 # ── network (the tests replace Net) ──────────────────────────────────────────────────────────────────────────────────
 class Net:
     def __init__(self, gh_token=""):
@@ -106,7 +115,7 @@ class Net:
         except urllib.error.HTTPError as e:
             return e.code, dict(e.headers or {}), e.read() if hasattr(e, "read") else b""
         except Exception as e:  # noqa: BLE001 — every transport failure is a NOT CHECKED, named
-            raise SourceError("%s: %s" % (url.split("?")[0], e))
+            raise Transient("%s: %s" % (url.split("?")[0], e))
 
     def post_json(self, url, body, timeout=30):
         self._count(url)
@@ -188,16 +197,32 @@ def patched_for(text, ours):
 
 # A repo whose advisory list covers more than the program we run: only these package names count.
 PACKAGE_ONLY = {"nextcloud/security-advisories": re.compile(r"^server$", re.I)}
+# A repo whose advisories also name OTHER programs we do not run (R-1310, read from every repo's list 2026-11-02). A notice
+# naming only these is not ours; a notice naming these AND the pinned program is. A new name is listed until it is added
+# here — the safe direction. Pinned: test W16.
+PACKAGE_NOT_OURS = {
+    "go-gitea/gitea": re.compile(r"gitea\.com/gitea/runner", re.I),
+    "home-assistant/core": re.compile(r"Companion|Operating System|Supervisor|home-assistant-js-websocket", re.I),
+    "advplyr/audiobookshelf": re.compile(r"audiobookshelf-app", re.I),
+    "go-vikunja/vikunja": re.compile(r"vikunja-desktop|code\.vikunja\.io/veans", re.I),
+    "actualbudget/actual": re.compile(r"@actual-app/cli", re.I),
+    "Termix-SSH/Termix": re.compile(r"@termix-plugin", re.I),
+    "louislam/uptime-kuma": re.compile(r"Rootless Docker Image only|louislam/dockge", re.I),
+    "PrivateBin/PrivateBin": re.compile(r"privatebin/(?:gcs|nginx-fpm-alpine|s3|unit-\S+)", re.I),
+}
 
 
 def ghsa_affects(adv, ours, repo=""):
     """-> (affected True/False/None, fixed text). None = the range is unreadable (listed, never dropped)."""
     verdicts, fixed = [], []
-    only = PACKAGE_ONLY.get(repo)
+    only, not_ours, skipped = PACKAGE_ONLY.get(repo), PACKAGE_NOT_OURS.get(repo), False
     for vul in adv.get("vulnerabilities") or []:
         pkg = (vul.get("package") or {}).get("name") or ""
         if only and not only.search(pkg.strip()):
             continue
+        if not_ours and not_ours.search(pkg):
+            skipped = True
+            continue
         # A GitHub Actions advisory is about the project's own CI workflow, never the image we run (R-1310: Home
         # Assistant's GHSA-jff5, 2023). Pinned: test W9.
         if ((vul.get("package") or {}).get("ecosystem") or "").lower() == "actions":
@@ -221,7 +246,7 @@ def ghsa_affects(adv, ours, repo=""):
     if any(v is True for v in verdicts):
         return True, ", ".join(sorted(set(fixed)))
     if not verdicts or all(v is False for v in verdicts):
-        return False if verdicts else (None if not only else False), "" if verdicts else "the notice names no version range"
+        return False if verdicts or skipped else (None if not only else False), "" if verdicts else "the notice names no version range"
     return None, "the range is unreadable — check by hand"
 
 
@@ -287,12 +312,20 @@ def cached_runtime_reader(cache, resolve=None, read=None):
             stats[key] += n
 
     def reader(ref):
-        try:
-            digest = resolve(ref)
-        except image_runtime.Unreadable as e:
-            return {"ref": ref, "unchecked": [str(e)]}
-        except Exception as e:  # noqa: BLE001 — a network error is a NOT CHECKED, never a crash of the whole watch
-            return {"ref": ref, "unchecked": ["%s: %s" % (type(e).__name__, str(e)[:120])]}
+        digest = None
+        for attempt in (1, 2):   # a read timeout gets one more try (R-1334)
+            try:
+                digest = resolve(ref)
+                break
+            except image_runtime.Unreadable as e:
+                # counted: the log's RUNTIME line said not-checked=0 for six nights while this path failed (R-1334)
+                count("not_checked")
+                return {"ref": ref, "unchecked": [str(e)]}
+            except Exception as e:  # noqa: BLE001 — a network error is a NOT CHECKED, never a crash of the whole watch
+                if attempt == 2:
+                    count("not_checked")
+                    return {"ref": ref, "unchecked": ["%s: %s" % (type(e).__name__, str(e)[:120])]}
+                time.sleep(5)
         if digest and digest in cache:
             hit = dict(cache[digest], ref=ref)
             hit["_used"] = True
@@ -324,6 +357,19 @@ HUB_TRIES = 4
 HUB_GAP_S = 1.0
 HUB_BACKOFF_S = (10, 30, 60)
 TAG_CACHE_S = 24 * 3600
+# A read timeout is tried again (R-1334: 1-3 pins a night failed that way on 2026-10-29..11-02, a different pin each
+# night, and were read fine the night after). The budget stops a real outage from stretching the job past its limit.
+TRANSIENT_TRIES = 3
+TRANSIENT_WAIT_S = (5, 15)
+TRANSIENT_BUDGET = 30
+# A pin the watch can never read, stated here with the ONE answer it gives: it is a GAP with this reason, not a failed
+# source. Any other answer (a timeout, a 500) is still NOT CHECKED, and an unlisted 401 too — a Docker Hub outage must
+# never read as a gap. Earned 2026-11-02 (R-1334): this one pin and our six private images held the „registry:docker.io"
+# and „runtime" streaks open for a week, so no later outage of either source could have mailed. Pinned: test W15.
+STATED_UNREADABLE = {
+    "minio/minio": ("HTTP 401", "Docker Hub answers HTTP 401 for its tags and its manifest: the image is no longer published "
+                                "there (seen 2026-10-28, R-1328). No notice can be matched; the homelab session moves the pin"),
+}
 
 
 class Sources:
@@ -335,10 +381,38 @@ class Sources:
         self.now = time.time() if now is None else now
         self.sleep, self.clock = sleep or time.sleep, clock or time.monotonic
         self._hub_lock, self._hub_last = threading.Lock(), None
-        self.tag_stats = {"from_cache": 0, "fetched": 0, "retried_429": 0}
+        self.tag_stats = {"from_cache": 0, "fetched": 0, "retried_429": 0, "retried_timeout": 0}
+        self._transient_left = TRANSIENT_BUDGET
+
+    def _may_retry_transient(self):
+        with self._hub_lock:
+            if self._transient_left <= 0:
+                return False
+            self._transient_left -= 1
+            self.tag_stats["retried_timeout"] += 1
+            return True
+
+    def get(self, url, headers=None):
+        """A GET to any source but Docker Hub: a read timeout is tried again, within tonight's budget."""
+        for i in range(TRANSIENT_TRIES):
+            try:
+                st, h, body = self.net.get(url, headers)
+            except Transient:
+                if i == TRANSIENT_TRIES - 1 or not self._may_retry_transient():
+                    raise
+                self.sleep(TRANSIENT_WAIT_S[min(i, len(TRANSIENT_WAIT_S) - 1)])
+                continue
+            hh = h or {}
+            # a 429 with quota left is a burst refusal (ghcr.io, 2026-11-02 by hand): wait and try again; GitHub's spent quota
+            # (X-RateLimit-Remaining: 0) is the caller's to read
+            spent = str(hh.get("X-RateLimit-Remaining", hh.get("x-ratelimit-remaining", ""))) == "0"
+            if st != 429 or spent or i == TRANSIENT_TRIES - 1 or not self._may_retry_transient():
+                return st, h, body
+            ra = str(hh.get("Retry-After", hh.get("retry-after", "")))
+            self.sleep(min(int(ra), 120) if ra.isdigit() else TRANSIENT_WAIT_S[min(i, len(TRANSIENT_WAIT_S) - 1)])
 
     def hub_get(self, url, headers=None):
-        """A GET to Docker Hub: spaced across the workers, a 429 retried. Returns the last answer."""
+        """A GET to Docker Hub: spaced across the workers, a 429 or a read timeout retried. Returns the last answer."""
         for i in range(HUB_TRIES):
             with self._hub_lock:
                 if self._hub_last is not None:
@@ -346,7 +420,13 @@ class Sources:
                     if wait > 0:
                         self.sleep(wait)
                 self._hub_last = self.clock()
-            st, h, body = self.net.get(url, headers)
+            try:
+                st, h, body = self.net.get(url, headers)
+            except Transient:
+                if i == HUB_TRIES - 1 or not self._may_retry_transient():
+                    raise
+                self.sleep(HUB_BACKOFF_S[min(i, len(HUB_BACKOFF_S) - 1)])
+                continue
             if st != 429 or i == HUB_TRIES - 1:
                 return st, h, body
             self.tag_stats["retried_429"] += 1
@@ -365,7 +445,7 @@ class Sources:
         if self.net.gh_token:
             headers["Authorization"] = "Bearer %s" % self.net.gh_token
         for _ in range(5):
-            st, h, body = self.net.get(url, headers)
+            st, h, body = self.get(url, headers)
             if st in (403, 429) and str(h.get("X-RateLimit-Remaining", h.get("x-ratelimit-remaining", ""))) == "0":
                 self.rate_limited = True
                 raise SourceError("GitHub API rate limit (%s)" % st)
@@ -382,7 +462,7 @@ class Sources:
     def release_feed(self, repo):
         if repo in self._rel:
             return self._rel[repo]
-        st, _h, body = self.net.get("https://github.com/%s/releases.atom" % repo)
+        st, _h, body = self.get("https://github.com/%s/releases.atom" % repo)
         if st != 200:
             raise SourceError("release feed of %s: HTTP %s" % (repo, st))
         text = body.decode("utf-8", "replace")
@@ -401,7 +481,7 @@ class Sources:
     def feed(self, name):
         if name in self._feed:
             return self._feed[name]
-        st, _h, body = self.net.get(FEEDS[name]["url"])
+        st, _h, body = self.get(FEEDS[name]["url"])
         if st != 200:
             raise SourceError("feed %s: HTTP %s" % (name, st))
         text = body.decode("utf-8", "replace")
@@ -419,7 +499,7 @@ class Sources:
     def vulnnotes(self, name):
         if name in self._notes:
             return self._notes[name]
-        st, _h, body = self.net.get(VULNNOTES[name]["url"])
+        st, _h, body = self.get(VULNNOTES[name]["url"])
         if st != 200:
             raise SourceError("release notes %s: HTTP %s" % (name, st))
         secs, cur = {}, None
@@ -480,13 +560,13 @@ class Sources:
                 if not url:
                     break
         elif reg == "ghcr.io":
-            st, _h, body = self.net.get("https://ghcr.io/token?scope=repository:%s:pull" % path)
+            st, _h, body = self.get("https://ghcr.io/token?scope=repository:%s:pull" % path)
             if st != 200:
                 raise SourceError("ghcr.io token for %s: HTTP %s" % (path, st))
             tok = json.loads(body).get("token", "")
             url = "https://ghcr.io/v2/%s/tags/list?n=10000" % path
             for _ in range(50):   # ghcr.io PAGES its list (Link: rel="next") — page 1 alone missed every newer tag (R-1327)
-                st, h, body = self.net.get(url, {"Authorization": "Bearer %s" % tok})
+                st, h, body = self.get(url, {"Authorization": "Bearer %s" % tok})
                 if st != 200:
                     raise SourceError("ghcr.io tags of %s: HTTP %s" % (path, st))
                 names += json.loads(body).get("tags") or []
@@ -517,6 +597,10 @@ def registry_of(image):
 
 
 # ── one pin ──────────────────────────────────────────────────────────────────────────────────────────────────────────
+def is_date_build(v):
+    return len(v) == 3 and 20000101 <= v[2] <= 20991231
+
+
 def newest_in_channel(tag, names):
     ours = version_of(tag)
     if not ours or not names:
@@ -529,6 +613,8 @@ def newest_in_channel(tag, names):
         v = version_of(n)
         if not v or len(v) != len(ours):
             continue
+        if is_date_build(v) and not is_date_build(ours):
+            continue   # linuxserver's 26.09.20260929 is a rebuild date, not a version above 26.09.2 (R-1310, test W16)
         if same_channel(ours, v):
             if vcmp(v, ours) > 0 and (best is None or vcmp(v, version_of(best)) > 0):
                 best = n
@@ -570,15 +656,25 @@ def check_item(it, src, today):
                                      "fixed": fixed, "link": a.get("html_url") or "",
                                      "unsure": hit is None, "published": (a.get("published_at") or "")[:10]})
             if kind == "github" and not floating:
-                for e in src.release_feed(arg):
+                named = {a["id"] for a in advs}
+                # oldest first, so a GHSA named by two releases reads as fixed in the lower one
+                for e in sorted(src.release_feed(arg), key=lambda r: (version_of(r["tag"]) or ()) + (0,) * 6):
                     v = version_of(e["tag"])
                     if not v or PRERELEASE.search(e["tag"]) or vcmp(v, ours) <= 0 or not same_channel(ours, v):
                         continue
-                    m = STRONG.search(e["content"]) or STRONG.search(e["title"])
-                    if m:
-                        line = next((ln.strip() for ln in re.split(r"[\n.]", e["content"]) if STRONG.search(ln)), m.group(0))
+                    lines = [ln.strip() for ln in re.split(r"[\n.]", e["content"]) + [e["title"]]
+                             if STRONG.search(ln) and not WORD_ONLY.search(ln)]
+                    ghsas = []
+                    for ln in lines:   # every GHSA a release names is its own entry (R-1310: Tandoor 2.7.0 named eight)
+                        ghsas += [("GHSA" + g[4:].lower(), ln) for g in GHSA.findall(ln)]
+                    for gid, ln in ghsas:
+                        if gid not in named:
+                            named.add(gid)
+                            advs.append({"id": gid, "source": "release notes", "severity": "", "summary": ln[:200],
+                                         "fixed": e["tag"], "link": e["link"], "unsure": False, "published": ""})
+                    if lines and not ghsas:
                         advs.append({"id": "release:%s@%s" % (arg, e["tag"]), "source": "release notes",
-                                     "severity": "", "summary": line[:200], "fixed": e["tag"], "link": e["link"],
+                                     "severity": "", "summary": lines[0][:200], "fixed": e["tag"], "link": e["link"],
                                      "unsure": False, "published": ""})
             elif kind == "osv" and not floating:
                 ver = ".".join(str(x) for x in ours)
@@ -637,15 +733,26 @@ def check_item(it, src, today):
             else:
                 behind, newer_major = newest_in_channel(it["tag"], names)
         except SourceError as e:
-            nc.append({"source": "registry:%s" % registry_of(it["image"])[0], "why": str(e)})
+            if stated_unreadable(it, str(e)):
+                it["gap_why"] = stated_unreadable(it, str(e))
+                notes.append("registry: %s" % e)
+            else:
+                nc.append({"source": "registry:%s" % registry_of(it["image"])[0], "why": str(e)})
     if newer_major:
         notes.append("a newer major exists: %s" % newer_major)
     if getattr(src, "runtime", None) and not it.get("no_registry") and it.get("image") and it.get("tag"):
         res = src.runtime("%s:%s" % (it["image"], it["tag"]))
         unchecked = res.get("unchecked") or []
         no_route = [u for u in unchecked if u.startswith("registry ") and "no anonymous token route" in u]
-        if no_route and it["group"] != "dooplex":
+        if no_route:
             notes.append("runtime: %s" % no_route[0])
+            if it["group"] == "dooplex":
+                # our own private registry: the CI has no read of it, tonight or ever. A stated hole, not a failed source
+                # (R-1334: six such images kept the „runtime" streak open, so a real outage could not mail).
+                it["gap_why"] = "%s — our own build in the private registry: registry and runtime not read" % no_route[0]
+        elif unchecked and all(stated_unreadable(it, u) for u in unchecked):
+            it["gap_why"] = stated_unreadable(it, unchecked[0])
+            notes.append("runtime: %s" % unchecked[0])
         elif unchecked:
             nc.append({"source": "runtime", "why": "; ".join(unchecked)[:300]})
         advs += runtime_advisories(res, it["group"])
@@ -656,7 +763,7 @@ def check_item(it, src, today):
         it["class"] = "ADVISORY"
     elif nc:
         it["class"] = "NOT CHECKED"
-    elif "gap" in kinds and not [k for k in kinds if k != "gap"]:
+    elif it.get("gap_why") or ("gap" in kinds and not [k for k in kinds if k != "gap"]):
         it["class"] = "GAP"
     elif "self" in kinds and len(kinds) == 1:
         it["class"] = "SELF"
@@ -669,6 +776,18 @@ def check_item(it, src, today):
     return it
 
 
+def stated_unreadable(it, why):
+    """The stated reason when this failure is the one answer this pin is known to give (STATED_UNREADABLE), else None."""
+    known = STATED_UNREADABLE.get(it.get("image"))
+    return known[1] if known and known[0] in why else None
+
+
+def failed_lines(items):
+    """Every failed source of every pin. The table shows one class per pin, so an ADVISORY pin hid its failed source."""
+    return ["FAILED-SOURCE %s %s:%s — %s" % (f["source"], it["image"], it["tag"], f["why"][:200])
+            for it in items for f in it.get("not_checked", [])]
+
+
 def source_name(kind, arg):
     return {"github": "github", "advisories": "github", "osv": "osv", "feed": "feed:" + arg,
             "vulnnotes": "vulnnotes:" + arg}.get(kind, kind)
@@ -840,8 +959,8 @@ def weekly_mail(items, today):
     lines += block("Open advisories (mailed already):", sorted({"%s %s — %s" % (it["image"], it["tag"], a["id"])
                                                               for it in items for a in it.get("advisories", [])}))
     lines += block("No machine-readable source (GAP) — not watched for notices:",
-                   sorted({"%s — %s" % (it["image"], next((v for k, v in it.get("sources", []) if k == "gap"),
-                                                         "; ".join(it.get("notes", []))))
+                   sorted({"%s — %s" % (it["image"], it.get("gap_why") or next((v for k, v in it.get("sources", []) if k == "gap"),
+                                                                                     "; ".join(it.get("notes", []))))
                            for it in items if it.get("class") == "GAP"}))
     lines += block("Floating tags (no exact version to match a notice against):",
                    sorted({"%s:%s (%s)" % (it["image"], it["tag"], it["owner"]) for it in items if it.get("class") == "FLOATING"}))
@@ -951,13 +1070,19 @@ def main(argv):
     for it in items:
         counts[it["class"]] = counts.get(it["class"], 0) + 1
     print("COUNTS " + " ".join("%s=%d" % (k.replace(" ", "_"), v) for k, v in sorted(counts.items())))
+    failed = failed_lines(items)
+    for line in failed:
+        print(line)
+    print("FAILED-SOURCES %d" % len(failed))
     if rreader:
         st = rreader.stats
-        print("RUNTIME read=%d (%.0f s) from-cache=%d not-checked=%d zstd-decoder=%s" % (
-            st["read"], st["read_s"], st["cached"], st["not_checked"],
+        nc_rt = sum(1 for it in items for f in it.get("not_checked", []) if f["source"] == "runtime")
+        print("RUNTIME read=%d (%.0f s) from-cache=%d not-checked=%d stated-unreadable=%d zstd-decoder=%s" % (
+            st["read"], st["read_s"], st["cached"], nc_rt, max(st["not_checked"] - nc_rt, 0),
             image_runtime.zstd_decoder_how() if st["read"] else "not needed"))
     ts = run.sources.tag_stats
-    print("TAGS fetched=%d from-cache=%d retried-429=%d (cache %s)" % (ts["fetched"], ts["from_cache"], ts["retried_429"], tpath))
+    print("TAGS fetched=%d from-cache=%d retried-429=%d retried-timeout=%d (cache %s)" % (
+        ts["fetched"], ts["from_cache"], ts["retried_429"], ts["retried_timeout"], tpath))
     print("REQUESTS " + " ".join("%s=%d" % kv for kv in sorted(net.requests.items())))
     state_path = opts.get("state")
     state, missing = {}, True