#!/usr/bin/env python3 """Measure upstream drift for app-catalog-felhom.eu image pins. Read-only, DooPlex-only. Reuses the auth-handling approach of scripts/check-image-resolvable.py (per-registry anonymous bearer-token flow against the Docker Registry v2 API), but instead of `docker manifest inspect` per-ref, lists the full upstream tag catalogue so we can compare the pinned tag against the newest one available, and classify the step as same-major (minor/patch) or major. """ import json import re import sys import time from pathlib import Path import requests SCRATCH = Path(".") SESSION = requests.Session() SESSION.headers.update({"User-Agent": "felhom-catalog-drift-audit/1.0 (read-only measurement)"}) def get_bearer_token(www_auth: str) -> str: assert www_auth.startswith("Bearer ") parts = www_auth[len("Bearer "):].split(",") d = {} for p in parts: k, v = p.split("=", 1) d[k] = v.strip('"') r = SESSION.get(d["realm"], params={"service": d.get("service"), "scope": d.get("scope")}, timeout=20) r.raise_for_status() return r.json()["token"] def tags_list_all(host: str, repo: str, max_pages=1000): """Full tag list via Docker Registry v2 API, following RFC5988 Link pagination.""" url = f"https://{host}/v2/{repo}/tags/list" params = {"n": 1000} r = SESSION.get(url, params=params, timeout=30) headers = {} if r.status_code == 401: tok = get_bearer_token(r.headers["WWW-Authenticate"]) headers = {"Authorization": f"Bearer {tok}"} r = SESSION.get(url, params=params, headers=headers, timeout=30) r.raise_for_status() j = r.json() tags = list(j.get("tags") or []) link = r.headers.get("Link") pages = 1 while link and pages < max_pages: m = re.search(r'<([^>]+)>;\s*rel="next"', link) if not m: break nexturl = m.group(1) if nexturl.startswith("/"): nexturl = f"https://{host}{nexturl}" r = SESSION.get(nexturl, headers=headers, timeout=30) r.raise_for_status() j = r.json() tags.extend(j.get("tags") or []) link = r.headers.get("Link") pages += 1 return tags def manifest_digest(host: str, repo: str, ref: str): """Resolve one ref's manifest digest (for floating-tag drift: 'has the tag moved'). Read-only.""" url = f"https://{host}/v2/{repo}/manifests/{ref}" accept = ("application/vnd.docker.distribution.manifest.v2+json," "application/vnd.docker.distribution.manifest.list.v2+json," "application/vnd.oci.image.manifest.v1+json," "application/vnd.oci.image.index.v1+json") r = SESSION.head(url, headers={"Accept": accept}, timeout=30) if r.status_code == 401: tok = get_bearer_token(r.headers["WWW-Authenticate"]) r = SESSION.head(url, headers={"Accept": accept, "Authorization": f"Bearer {tok}"}, timeout=30) if r.status_code != 200: return None, f"HTTP {r.status_code}" return r.headers.get("Docker-Content-Digest"), None # host resolution per repo prefix, mirroring how these refs are actually pulled def resolve_host_repo(ref_repo: str): if ref_repo.startswith("ghcr.io/"): return "ghcr.io", ref_repo[len("ghcr.io/"):] if ref_repo.startswith("lscr.io/"): return "lscr.io", ref_repo[len("lscr.io/"):] if ref_repo.startswith("registry.gitlab.com/"): return "registry.gitlab.com", ref_repo[len("registry.gitlab.com/"):] if ref_repo.startswith("gitea.dooplex.hu/"): return "gitea.dooplex.hu", ref_repo[len("gitea.dooplex.hu/"):] if ref_repo.startswith("quay.io/"): return "quay.io", ref_repo[len("quay.io/"):] # Docker Hub if "/" not in ref_repo: return "registry-1.docker.io", f"library/{ref_repo}" return "registry-1.docker.io", ref_repo VERSION_CORE_RE = re.compile(r'(\d+(?:\.\d+){1,3})') UNSTABLE_MARKERS = ("rc", "beta", "alpha", "dev", "nightly", "canary", "edge", "preview", "snapshot", "-pr", "test") def split_tag(tag: str): """(prefix, core_version_tuple, suffix) — first maximal dotted-numeric run of len>=2 is the core.""" m = VERSION_CORE_RE.search(tag) if not m: return None prefix = tag[:m.start()] core = tuple(int(x) for x in m.group(1).split(".")) suffix = tag[m.end():] return prefix, core, suffix def is_unstable(tag: str) -> bool: low = tag.lower() return any(mk in low for mk in UNSTABLE_MARKERS) def newest_matching(all_tags, current_tag): """Find the newest tag sharing the current tag's prefix/suffix 'shape', by core version tuple.""" cur = split_tag(current_tag) if not cur: return None, "current tag has no parseable version core" cur_prefix, cur_core, cur_suffix = cur # suffix "shape": if suffix contains a long hex-looking token, treat it as a wildcard hash def suffix_shape(suf): if re.search(r'[0-9a-f]{7,40}', suf.lower()) and re.search(r'[a-f]', suf.lower()): return re.sub(r'[0-9a-f]{7,40}', '', suf.lower()) return suf cur_shape = suffix_shape(cur_suffix) best = None best_core = None for t in all_tags: if t == current_tag: continue if is_unstable(t): continue parsed = split_tag(t) if not parsed: continue p, core, suf = parsed if p != cur_prefix: continue if suffix_shape(suf) != cur_shape: continue if len(core) != len(cur_core): continue # Some registries (linuxserver.io/lscr.io in particular) publish EXTRA tags for a # same-app-version base-image rebuild, shaped like the release tag plus a YYYYMMDD-ish # trailing component (e.g. bookstack 26.05.2 coexists with 26.05.20260608). That is not a # newer application release — it is a rebuild of an existing one — so any component that # looks like a calendar date (>= 20000, comfortably above any real release counter we saw # across all 58 pinned-semver images, all of which stayed under 10000) is excluded as a # candidate "newest" rather than misread as a huge version jump. if any(c >= 20000 for c in core): continue if best_core is None or core > best_core: best_core = core best = t if best is None: return None, "no matching newer tag found (same prefix/suffix shape)" if best_core <= cur_core: return None, "up to date (no tag newer than current within the same shape)" return (best, best_core, cur_core), None def newest_same_major(all_tags, current_tag): cur = split_tag(current_tag) if not cur: return None p0, c0, s0 = cur best=None; bc=None for t in all_tags: if t == current_tag or is_unstable(t): continue pr = split_tag(t) if not pr: continue p, c, s = pr if p != p0 or s != s0 or len(c) != len(c0) or c[0] != c0[0]: continue if any(x >= 20000 for x in c): continue if c <= c0: continue if bc is None or c > bc: bc=c; best=t return best def main(): import glob, yaml pins = {} for f in sorted(glob.glob("/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/templates/*/docker-compose.yml")): app = f.split("/")[-2] d = yaml.safe_load(open(f)) for svc, sd in (d.get("services") or {}).items(): img = (sd or {}).get("image") if not img or "${" in img: continue repo, _, tag = img.rpartition(":") if ":" in img.split("/")[-1] else (img, "", "latest") pins.setdefault(img, {"repo": repo, "tag": tag, "apps": []})["apps"].append(f"{app}/{svc}") out = {} for i,(ref,p) in enumerate(sorted(pins.items())): e = dict(ref=ref, **p) host, hrepo = resolve_host_repo(p["repo"]) if p["repo"].startswith("gitea.dooplex.hu"): e["status"]="internal"; out[ref]=e; continue try: tags = tags_list_all(host, hrepo) res, err = newest_matching(tags, p["tag"]) e["newest_any"] = res[0] if res else None e["newest_same_major"] = newest_same_major(tags, p["tag"]) e["status"] = "behind" if e["newest_any"] else "current-or-unparsed" except Exception as ex: e["status"]="error"; e["error"]=f"{type(ex).__name__}: {ex}" print(f"[{i+1}/{len(pins)}] {ref}: same-major={e.get('newest_same_major')} any={e.get('newest_any')} {e.get('error','')}", flush=True) out[ref]=e time.sleep(0.15) json.dump(out, open("02-drift.json","w"), indent=2) if __name__ == "__main__": main()