felhom.eu/documentation/audits/catalog-currency-2026-09-30/00-currency.py

#!/usr/bin/env python3
"""Catalog currency audit, 2026-09-30. Read-only, DooPlex-only, anonymous public registry API.

Adapted from documentation/audits/update-arc-2026-09-21/00-drift.py (same bearer-token flow and the
same Link pagination for /v2/<repo>/tags/list; same lscr.io date-rebuild exclusion; same unstable
markers). Changes, and why:
  - reads the pins straight from the catalog checkout (scripts/ladder.py images_in — the same reading
    the gates make), not from a hand-built pins.json;
  - one generic tag tokenizer for every pin (numbers vs the text between them; a long hex token is a
    build hash, a wildcard). Two tags are comparable only when the text between their numbers is
    identical and they have the same count of numbers. So `postgres:16-alpine` compares only with
    `<N>-alpine`, `nextcloud:34.0.4-apache` only with `<a>.<b>.<c>-apache`;
  - reports BOTH the newest within the pin's major (first number equal) and the newest at all;
  - classifies a pin as FLOATING when it names a version LINE, not an exact release: fewer than three
    version numbers in the tag's leading version run (e.g. `mariadb:12.3`, `postgres:16-alpine`,
    `redis:7-alpine`, `postgis:16-3.5-alpine`, immich's `postgres:16-vectorchord...`);
  - for floating pins, also resolves the tag's CURRENT digest (HEAD, no pull, no rate-limit count on
    Docker Hub) and compares it with the digest the app's ladder recorded when the step was tested.
Nothing is pulled; nothing is written outside the output directory.

Usage: python3 00-currency.py <catalog-root> <outdir>
"""
import json
import os
import re
import sys
import time

import requests

CAT = sys.argv[1]
OUT = sys.argv[2]
sys.path.insert(0, os.path.join(CAT, "scripts"))
import ladder  # noqa: E402

S = requests.Session()
S.headers.update({"User-Agent": "felhom-catalog-currency-audit/1.0 (read-only measurement)"})


def bearer(www):
    parts = www[len("Bearer "):].split(",")
    d = {}
    for p in parts:
        k, v = p.split("=", 1)
        d[k.strip()] = v.strip('"')
    r = S.get(d["realm"], params={"service": d.get("service"), "scope": d.get("scope")}, timeout=20)
    r.raise_for_status()
    j = r.json()
    return j.get("token") or j.get("access_token")


def get(url, headers=None, method="GET", params=None, _tok={}):
    headers = dict(headers or {})
    host = url.split("/")[2]
    if host in _tok:
        headers["Authorization"] = "Bearer " + _tok[host]
    r = S.request(method, url, headers=headers, params=params, timeout=30)
    if r.status_code == 401 and "WWW-Authenticate" in r.headers:
        tok = bearer(r.headers["WWW-Authenticate"])
        headers["Authorization"] = "Bearer " + tok
        r = S.request(method, url, headers=headers, params=params, timeout=30)
    return r


def tags_all(host, repo):
    url = f"https://{host}/v2/{repo}/tags/list"
    r = get(url, params={"n": 1000})
    r.raise_for_status()
    auth = r.request.headers.get("Authorization")
    tags = list(r.json().get("tags") or [])
    link = r.headers.get("Link")
    pages = 1
    while link and pages < 1000:
        m = re.search(r'<([^>]+)>;\s*rel="next"', link)
        if not m:
            break
        nxt = m.group(1)
        if nxt.startswith("/"):
            nxt = f"https://{host}{nxt}"
        r = S.get(nxt, headers={"Authorization": auth} if auth else {}, timeout=30)
        r.raise_for_status()
        tags.extend(r.json().get("tags") or [])
        link = r.headers.get("Link")
        pages += 1
    return tags


ACCEPT = ("application/vnd.docker.distribution.manifest.v2+json,"
          "application/vnd.docker.distribution.manifest.list.v2+json,"
          "application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json")


def digest(host, repo, tag):
    r = get(f"https://{host}/v2/{repo}/manifests/{tag}", headers={"Accept": ACCEPT}, method="HEAD")
    if r.status_code != 200:
        return None, f"HTTP {r.status_code}"
    return r.headers.get("Docker-Content-Digest"), None


def host_repo(ref_repo):
    for h in ("ghcr.io", "lscr.io", "registry.gitlab.com", "gitea.dooplex.hu", "quay.io"):
        if ref_repo.startswith(h + "/"):
            return h, ref_repo[len(h) + 1:]
    if "/" not in ref_repo:
        return "registry-1.docker.io", "library/" + ref_repo
    return "registry-1.docker.io", ref_repo


def split_ref(ref):
    ref = ref.split("@")[0]
    slash = ref.rfind("/")
    colon = ref.rfind(":")
    if colon > slash:
        return ref[:colon], ref[colon + 1:]
    return ref, "latest"


UNSTABLE = ("rc", "beta", "alpha", "dev", "nightly", "canary", "edge", "preview", "snapshot", "-pr", "test", "unstable")
HASH_RE = re.compile(r"(?<![0-9a-z])(?=[0-9a-f]*[a-f])(?=[0-9a-f]*[0-9])[0-9a-f]{7,40}(?![0-9a-z])")


def tokenize(tag):
    """(shape, numbers) or None when the tag carries no number."""
    t = HASH_RE.sub("<hash>", tag.lower())
    parts = re.split(r"(\d+)", t)
    nums = tuple(int(p) for p in parts[1::2])
    if not nums:
        return None
    shape = tuple(parts[0::2])
    return shape, nums


def lead_run_len(tag):
    m = re.search(r"\d+(?:\.\d+)*", tag)
    return len(m.group(0).split(".")) if m else 0


def is_floating(tag):
    if not re.search(r"\d", tag):
        return True  # `latest`-style: no version at all
    return lead_run_len(tag) < 3


def unstable(tag):
    low = tag.lower()
    return any(mk in low for mk in UNSTABLE)


def newest(tags, cur):
    c = tokenize(cur)
    if not c:
        return None
    shape, nums = c
    best_all = best_maj = None
    for t in tags:
        if unstable(t) and not unstable(cur):
            continue
        p = tokenize(t)
        if not p or p[0] != shape or len(p[1]) != len(nums):
            continue
        if any(n >= 20000 for n in p[1]) and not any(n >= 20000 for n in nums):
            continue  # lscr.io-style date rebuild tag, not a release (see 00-drift.py)
        if best_all is None or p[1] > best_all[1]:
            best_all = (t, p[1])
        if p[1][0] == nums[0] and (best_maj is None or p[1] > best_maj[1]):
            best_maj = (t, p[1])
    return {"cur": nums, "all": best_all, "maj": best_maj}


def ladder_digests(app_dir):
    fy = os.path.join(app_dir, ".felhom.yml")
    entries, _r, _e = ladder.parse(open(fy, encoding="utf-8").read())
    if not entries:
        return {}, None
    head = entries[-1]
    return head.get("digest", {}), head.get("tested_at")


def main():
    tdir = os.path.join(CAT, "templates")
    rows = []
    cache = {}
    for app in sorted(os.listdir(tdir)):
        d = os.path.join(tdir, app)
        imgs = ladder.images_in(open(os.path.join(d, "docker-compose.yml"), encoding="utf-8").read())
        ldig, ltested = ladder_digests(d)
        for svc, ref in imgs.items():
            repo, tag = split_ref(ref)
            host, hrepo = host_repo(repo)
            row = {"app": app, "service": svc, "ref": ref, "repo": repo, "tag": tag, "host": host,
                   "floating": is_floating(tag)}
            if host == "gitea.dooplex.hu":
                row["status"] = "internal"
                rows.append(row)
                print(f"{app}/{svc}: internal image, not upstream")
                continue
            try:
                if (host, hrepo) not in cache:
                    cache[(host, hrepo)] = tags_all(host, hrepo)
                    time.sleep(0.15)
                tags = cache[(host, hrepo)]
                row["n_tags"] = len(tags)
                row["pinned_tag_listed"] = tag in tags
                res = newest(tags, tag)
                if res is None:
                    row["status"] = "unparsed"
                else:
                    row["status"] = "ok"
                    ba, bm = res["all"], res["maj"]
                    row["newest_all"] = ba[0] if ba else None
                    row["newest_major"] = bm[0] if bm else None
                    row["behind_within_major"] = bool(bm and bm[1] > res["cur"])
                    row["behind_across_major"] = bool(ba and ba[1][0] > res["cur"][0])
                    row["major"] = res["cur"][0]
                if row["floating"]:
                    dg, err = digest(host, hrepo, tag)
                    row["current_digest"] = dg
                    row["digest_error"] = err
                    row["ladder_digest"] = ldig.get(svc)
                    row["ladder_tested_at"] = ltested
                    if dg and ldig.get(svc):
                        row["moved_since_tested"] = dg != ldig.get(svc)
            except Exception as e:  # recorded, never guessed
                row["status"] = "error"
                row["error"] = f"{type(e).__name__}: {e}"[:300]
            rows.append(row)
            print(f"{app}/{svc}: {tag} -> maj={row.get('newest_major')} all={row.get('newest_all')} "
                  f"float={row['floating']} {row['status']} {row.get('error','')}")
    json.dump(rows, open(os.path.join(OUT, "01-currency-raw.json"), "w"), indent=1)
    print("rows", len(rows))


if __name__ == "__main__":
    main()