security watch: Docker Hub requests spaced, a 429 retried (NOT CHECKED only after the last try), tag lists cached 24 h per registry path on the runner (R-1334); W14

scripts/security_watch.py

@@ -52,6 +52,8 @@ import json
 import os
 import re
 import sys
+import threading
+import time
 import urllib.error
 import urllib.request
 import zlib
@@ -313,11 +315,44 @@ def cached_runtime_reader(cache, resolve=None, read=None):
     return reader
 
 
+# Docker Hub (R-1334): its tag API refused the 2026-10-28 night run (15 x HTTP 429) when 211 requests went out in about
+# 2 minutes from six workers; the 07:03 dry run sent 224 in 3.5 minutes and got none. So the requests to Docker Hub are
+# SPACED (one per HUB_GAP_S, across the workers), a 429 is RETRIED (its Retry-After, capped) and is NOT CHECKED only after
+# the last try, and a tag list is kept for TAG_CACHE_S per registry path (two names of one path fetch once).
+HUB_HOSTS = ("hub.docker.com", "registry-1.docker.io", "auth.docker.io")
+HUB_TRIES = 4
+HUB_GAP_S = 1.0
+HUB_BACKOFF_S = (10, 30, 60)
+TAG_CACHE_S = 24 * 3600
+
+
 class Sources:
-    def __init__(self, net, today, rotate):
+    def __init__(self, net, today, rotate, tag_cache=None, now=None, sleep=None, clock=None):
         self.net, self.today, self.rotate = net, today, rotate
         self._adv, self._rel, self._feed, self._notes, self._osv, self._tags = {}, {}, {}, {}, {}, {}
         self.rate_limited = False
+        self.tag_cache = tag_cache if tag_cache is not None else {}
+        self.now = time.time() if now is None else now
+        self.sleep, self.clock = sleep or time.sleep, clock or time.monotonic
+        self._hub_lock, self._hub_last = threading.Lock(), None
+        self.tag_stats = {"from_cache": 0, "fetched": 0, "retried_429": 0}
+
+    def hub_get(self, url, headers=None):
+        """A GET to Docker Hub: spaced across the workers, a 429 retried. Returns the last answer."""
+        for i in range(HUB_TRIES):
+            with self._hub_lock:
+                if self._hub_last is not None:
+                    wait = HUB_GAP_S - (self.clock() - self._hub_last)
+                    if wait > 0:
+                        self.sleep(wait)
+                self._hub_last = self.clock()
+            st, h, body = self.net.get(url, headers)
+            if st != 429 or i == HUB_TRIES - 1:
+                return st, h, body
+            self.tag_stats["retried_429"] += 1
+            ra = str((h or {}).get("Retry-After", (h or {}).get("retry-after", "")))
+            self.sleep(min(int(ra), 120) if ra.isdigit() else HUB_BACKOFF_S[min(i, len(HUB_BACKOFF_S) - 1)])
+        return st, h, body
 
     def gh_advisories(self, repo):
         if repo in self._adv:
@@ -408,30 +443,37 @@ class Sources:
         return self._osv[key]
 
     def _registry_tags(self, path):
-        st, _h, body = self.net.get("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull" % path)
+        st, _h, body = self.hub_get("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull" % path)
         if st != 200:
             raise SourceError("Docker registry token for %s: HTTP %s" % (path, st))
         tok = json.loads(body).get("token", "")
-        st, _h, body = self.net.get("https://registry-1.docker.io/v2/%s/tags/list?n=10000" % path,
+        st, _h, body = self.hub_get("https://registry-1.docker.io/v2/%s/tags/list?n=10000" % path,
                                     {"Authorization": "Bearer %s" % tok})
         if st != 200:
             raise SourceError("Docker registry tags of %s: HTTP %s" % (path, st))
         return json.loads(body).get("tags") or []
 
     def tags(self, image):
-        if image in self._tags:
-            return self._tags[image]
         reg, path = registry_of(image)
+        key = "%s/%s" % (reg, path)
+        if key in self._tags:
+            return self._tags[key]
+        hit = self.tag_cache.get(key)
+        if hit and self.now - hit.get("at", 0) < TAG_CACHE_S and isinstance(hit.get("tags"), list):
+            self.tag_stats["from_cache"] += 1
+            self._tags[key] = hit["tags"]
+            return hit["tags"]
         names = []
         if reg == "docker.io":
             url = "https://hub.docker.com/v2/repositories/%s/tags?page_size=100&ordering=last_updated" % path
             for _ in range(3):
-                st, _h, body = self.net.get(url)
+                st, h, body = self.hub_get(url)
                 if st == 404 and not names:
                     names = self._registry_tags(path)  # the web API misses some repos the registry serves (plant-it)
                     break
                 if st != 200:
-                    raise SourceError("Docker Hub tags of %s: HTTP %s" % (path, st))
+                    raise SourceError("Docker Hub tags of %s: HTTP %s%s" % (
+                        path, st, " after %d tries" % HUB_TRIES if st == 429 else ""))
                 d = json.loads(body)
                 names += [t["name"] for t in d.get("results", [])]
                 url = d.get("next")
@@ -454,7 +496,10 @@ class Sources:
                 url = "https://ghcr.io" + nxt.group(1) if nxt.group(1).startswith("/") else nxt.group(1)
         else:
             names = None  # a registry the watch does not read (our own Gitea, GitLab): stated, not a failure
-        self._tags[image] = names
+        self._tags[key] = names
+        if names is not None:
+            self.tag_stats["fetched"] += 1
+            self.tag_cache[key] = {"at": self.now, "tags": names}
         return names
 
 
@@ -822,11 +867,12 @@ def send(mail, key, test=False):
 
 
 # ── the run ──────────────────────────────────────────────────────────────────────────────────────────────────────────
-def run(items, net, today, rotate, runtime=None):
+def run(items, net, today, rotate, runtime=None, tag_cache=None):
     """`runtime`: a reader (cached_runtime_reader) for the images' own runtime parts; None = not looked at (the tests
-    of W1–W9). main() always hands one."""
-    src = Sources(net, today, rotate)
+    of W1–W9). main() always hands one. `tag_cache`: {registry/path: {"at", "tags"}}, read and updated (R-1334)."""
+    src = Sources(net, today, rotate, tag_cache=tag_cache)
     src.runtime = runtime
+    run.sources = src
     # Box programs first, one by one (they share sources with nothing); apps in a small pool.
     box = [it for it in items if _grp(it) == 0]
     apps = [it for it in items if _grp(it) != 0]   # catalog apps and the DooPlex list share the pool
@@ -886,7 +932,19 @@ def main(argv):
         prior = json.load(open(state_path, encoding="utf-8"))
     rcache = {k: dict(v, _seen=False) for k, v in (prior.get("runtime_cache") or {}).items()}
     rreader = None if "no-runtime" in opts else cached_runtime_reader(rcache)
-    run(items, net, today, rotate=not token, runtime=rreader)
+    # the tag lists' 24-hour cache lives on the CI runner (beside the wheel cache), not in the committed state (R-1334)
+    tpath = opts.get("tag-cache") or os.path.join(os.path.dirname(image_runtime._wheel_cache()), "felhom-watch-tags.json")
+    try:
+        tcache = json.load(open(tpath, encoding="utf-8"))
+    except (OSError, ValueError):
+        tcache = {}
+    run(items, net, today, rotate=not token, runtime=rreader, tag_cache=tcache)
+    try:
+        os.makedirs(os.path.dirname(tpath), exist_ok=True)
+        cut = time.time() - TAG_CACHE_S
+        json.dump({k: v for k, v in tcache.items() if v.get("at", 0) > cut}, open(tpath, "w", encoding="utf-8"))
+    except OSError as e:
+        print("TAG CACHE not saved (%s): %s" % (tpath, e))
     for line in table(items):
         print(line)
     counts = {}
@@ -898,6 +956,8 @@ def main(argv):
         print("RUNTIME read=%d (%.0f s) from-cache=%d not-checked=%d zstd-decoder=%s" % (
             st["read"], st["read_s"], st["cached"], st["not_checked"],
             image_runtime.zstd_decoder_how() if st["read"] else "not needed"))
+    ts = run.sources.tag_stats
+    print("TAGS fetched=%d from-cache=%d retried-429=%d (cache %s)" % (ts["fetched"], ts["from_cache"], ts["retried_429"], tpath))
     print("REQUESTS " + " ".join("%s=%d" % kv for kv in sorted(net.requests.items())))
     state_path = opts.get("state")
     state, missing = {}, True