image reader waits out a registry 429 and tries again (3 tries); 08 + changelog: the dry-run measurement of the Docker Hub change (0 refused), two runners = two tag caches (R-1334)

scripts/CHANGELOG.md

@@ -7,6 +7,10 @@
   wheel cache (`--tag-cache=` overrides). The log gains `TAGS fetched=… from-cache=… retried-429=…`.
 - Measured before: night job 2446, 211 Docker Hub requests in ~2 min, 15 refused; dry job 2450, 224 in ~3.5 min, 0 refused.
 - Tests: W14 (7 checks; the 429 retry seen failing on the old code — the first 429 ended the image) — 71 checks.
+- Measured after: dry runs 2458 and 2459 (`fe677bae`): 0 Docker Hub 429, 0 retries; 210 requests over ~7 min. The two runs
+  ran on the two different CI runners, so the second found no tag cache (each runner keeps its own).
+- **`image_runtime.py`:** `_get` waits out a 429 (`Retry-After`, else 10/30 s) and tries again, `RETRY_TRIES` = 3 — run 2459
+  had two ghcr.io blob reads refused. `test_image_runtime.py` retry checks (red on the old `_get`: one call, no wait) — 21 checks.
 
 ## 2026-10-28 — the watch remembers a notice per GROUP: a catalog pin and a DooPlex pin of one image are two entries
 

scripts/image_runtime.py

@@ -176,15 +176,25 @@ def split_ref(ref):
     return reg, path, tag
 
 
+RETRY_TRIES = 3          # a registry 429 is waited out (its Retry-After, capped, else 10/30 s) and tried again; only the
+RETRY_BACKOFF_S = (10, 30)   # last try's 429 is an answer (ghcr.io refused blob reads in back-to-back dry runs, 2026-10-28)
+
+
 def _get(url, headers=None, stream=False):
-    req = urllib.request.Request(url, headers=dict({"User-Agent": UA}, **(headers or {})))
-    try:
-        r = urllib.request.urlopen(req, timeout=60)
-    except urllib.error.HTTPError as e:
-        return e.code, e.headers, b""
-    if stream:
-        return r.status, r.headers, r
-    return r.status, r.headers, r.read()
+    import time
+    for i in range(RETRY_TRIES):
+        req = urllib.request.Request(url, headers=dict({"User-Agent": UA}, **(headers or {})))
+        try:
+            r = urllib.request.urlopen(req, timeout=60)
+        except urllib.error.HTTPError as e:
+            if e.code == 429 and i < RETRY_TRIES - 1:
+                ra = str((e.headers or {}).get("Retry-After", ""))
+                time.sleep(min(int(ra), 120) if ra.isdigit() else RETRY_BACKOFF_S[min(i, len(RETRY_BACKOFF_S) - 1)])
+                continue
+            return e.code, e.headers, b""
+        if stream:
+            return r.status, r.headers, r
+        return r.status, r.headers, r.read()
 
 
 class Registry:

scripts/test_image_runtime.py

@@ -134,6 +134,40 @@ def wheel_checks():
     sys.modules.pop("zstandard", None)
 
 
+def retry_checks():
+    """A registry 429 (ghcr.io refused blob reads in two back-to-back CI dry runs, 2026-10-28) is waited out and tried
+    again; the answer is a refusal only after the last try."""
+    import io, time, urllib.error, urllib.request
+    calls, slept = [], []
+    real_open, real_sleep = urllib.request.urlopen, time.sleep
+
+    class Resp(io.BytesIO):
+        status = 200
+        headers = {}
+
+    def fake(answers):
+        def op(req, timeout=60):
+            calls.append(req.full_url)
+            a = answers.pop(0) if len(answers) > 1 else answers[0]
+            if a == 429:
+                raise urllib.error.HTTPError(req.full_url, 429, "Too Many Requests", {"Retry-After": "3"}, io.BytesIO(b""))
+            return Resp(b"ok")
+        return op
+    try:
+        time.sleep = lambda s: slept.append(s)
+        urllib.request.urlopen = fake([429, 200])
+        st, _h, body = ir._get("https://ghcr.io/v2/x/blobs/sha256:a")
+        check("retry: a 429 then 200 -> the 200, after waiting its Retry-After", st == 200 and body == b"ok" and
+              len(calls) == 2 and 3 in slept, (st, calls, slept))
+        calls.clear()
+        urllib.request.urlopen = fake([429])
+        st, _h, body = ir._get("https://ghcr.io/v2/x/blobs/sha256:a")
+        check("retry: a 429 on every try -> 429 after %d tries, never a pass" % ir.RETRY_TRIES, st == 429 and
+              len(calls) == ir.RETRY_TRIES, (st, len(calls)))
+    finally:
+        urllib.request.urlopen, time.sleep = real_open, real_sleep
+
+
 def main():
     apk = ir.parse_apk("P:musl\nV:1.2.5-r10\n\nP:nodejs\nV:24.18.1-r0\n\nP:brotli-libs\nV:1.2.0-r1\n")
     check("apk database -> nodejs and brotli-libs", apk.get("nodejs") == "24.18.1-r0" and apk.get("brotli-libs") == "1.2.0-r1", apk)
@@ -163,6 +197,7 @@ def main():
                                                 "size": 10}], budget=0)
     check("a layer over the byte budget is a NOTE, never a silent pass", notes and ("budget" in notes[0] or "zstd" in notes[0]), notes)
     zstd_checks()
+    retry_checks()
     wheel_checks()
     d = tempfile.mkdtemp()
     open(os.path.join(d, "a.yaml"), "w").write("spec:\n  containers:\n  - image: rommapp/romm:5.4.0\n    name: x\n"