felhom.eu/documentation/audits/night-2026-09-25/E/drift.py

#!/usr/bin/env python3
"""Measure upstream drift for app-catalog-felhom.eu image pins. Read-only, DooPlex-only.

Reuses the auth-handling approach of scripts/check-image-resolvable.py (per-registry anonymous
bearer-token flow against the Docker Registry v2 API), but instead of `docker manifest inspect`
per-ref, lists the full upstream tag catalogue so we can compare the pinned tag against the newest
one available, and classify the step as same-major (minor/patch) or major.
"""
import json
import re
import sys
import time
from pathlib import Path

import requests

SCRATCH = Path(".")

SESSION = requests.Session()
SESSION.headers.update({"User-Agent": "felhom-catalog-drift-audit/1.0 (read-only measurement)"})


def get_bearer_token(www_auth: str) -> str:
    assert www_auth.startswith("Bearer ")
    parts = www_auth[len("Bearer "):].split(",")
    d = {}
    for p in parts:
        k, v = p.split("=", 1)
        d[k] = v.strip('"')
    r = SESSION.get(d["realm"], params={"service": d.get("service"), "scope": d.get("scope")}, timeout=20)
    r.raise_for_status()
    return r.json()["token"]


def tags_list_all(host: str, repo: str, max_pages=1000):
    """Full tag list via Docker Registry v2 API, following RFC5988 Link pagination."""
    url = f"https://{host}/v2/{repo}/tags/list"
    params = {"n": 1000}
    r = SESSION.get(url, params=params, timeout=30)
    headers = {}
    if r.status_code == 401:
        tok = get_bearer_token(r.headers["WWW-Authenticate"])
        headers = {"Authorization": f"Bearer {tok}"}
        r = SESSION.get(url, params=params, headers=headers, timeout=30)
    r.raise_for_status()
    j = r.json()
    tags = list(j.get("tags") or [])
    link = r.headers.get("Link")
    pages = 1
    while link and pages < max_pages:
        m = re.search(r'<([^>]+)>;\s*rel="next"', link)
        if not m:
            break
        nexturl = m.group(1)
        if nexturl.startswith("/"):
            nexturl = f"https://{host}{nexturl}"
        r = SESSION.get(nexturl, headers=headers, timeout=30)
        r.raise_for_status()
        j = r.json()
        tags.extend(j.get("tags") or [])
        link = r.headers.get("Link")
        pages += 1
    return tags


def manifest_digest(host: str, repo: str, ref: str):
    """Resolve one ref's manifest digest (for floating-tag drift: 'has the tag moved'). Read-only."""
    url = f"https://{host}/v2/{repo}/manifests/{ref}"
    accept = ("application/vnd.docker.distribution.manifest.v2+json,"
              "application/vnd.docker.distribution.manifest.list.v2+json,"
              "application/vnd.oci.image.manifest.v1+json,"
              "application/vnd.oci.image.index.v1+json")
    r = SESSION.head(url, headers={"Accept": accept}, timeout=30)
    if r.status_code == 401:
        tok = get_bearer_token(r.headers["WWW-Authenticate"])
        r = SESSION.head(url, headers={"Accept": accept, "Authorization": f"Bearer {tok}"}, timeout=30)
    if r.status_code != 200:
        return None, f"HTTP {r.status_code}"
    return r.headers.get("Docker-Content-Digest"), None


# host resolution per repo prefix, mirroring how these refs are actually pulled
def resolve_host_repo(ref_repo: str):
    if ref_repo.startswith("ghcr.io/"):
        return "ghcr.io", ref_repo[len("ghcr.io/"):]
    if ref_repo.startswith("lscr.io/"):
        return "lscr.io", ref_repo[len("lscr.io/"):]
    if ref_repo.startswith("registry.gitlab.com/"):
        return "registry.gitlab.com", ref_repo[len("registry.gitlab.com/"):]
    if ref_repo.startswith("gitea.dooplex.hu/"):
        return "gitea.dooplex.hu", ref_repo[len("gitea.dooplex.hu/"):]
    if ref_repo.startswith("quay.io/"):
        return "quay.io", ref_repo[len("quay.io/"):]
    # Docker Hub
    if "/" not in ref_repo:
        return "registry-1.docker.io", f"library/{ref_repo}"
    return "registry-1.docker.io", ref_repo


VERSION_CORE_RE = re.compile(r'(\d+(?:\.\d+){1,3})')
UNSTABLE_MARKERS = ("rc", "beta", "alpha", "dev", "nightly", "canary", "edge", "preview", "snapshot", "-pr", "test")


def split_tag(tag: str):
    """(prefix, core_version_tuple, suffix) — first maximal dotted-numeric run of len>=2 is the core."""
    m = VERSION_CORE_RE.search(tag)
    if not m:
        return None
    prefix = tag[:m.start()]
    core = tuple(int(x) for x in m.group(1).split("."))
    suffix = tag[m.end():]
    return prefix, core, suffix


def is_unstable(tag: str) -> bool:
    low = tag.lower()
    return any(mk in low for mk in UNSTABLE_MARKERS)


def newest_matching(all_tags, current_tag):
    """Find the newest tag sharing the current tag's prefix/suffix 'shape', by core version tuple."""
    cur = split_tag(current_tag)
    if not cur:
        return None, "current tag has no parseable version core"
    cur_prefix, cur_core, cur_suffix = cur

    # suffix "shape": if suffix contains a long hex-looking token, treat it as a wildcard hash
    def suffix_shape(suf):
        if re.search(r'[0-9a-f]{7,40}', suf.lower()) and re.search(r'[a-f]', suf.lower()):
            return re.sub(r'[0-9a-f]{7,40}', '<hash>', suf.lower())
        return suf

    cur_shape = suffix_shape(cur_suffix)

    best = None
    best_core = None
    for t in all_tags:
        if t == current_tag:
            continue
        if is_unstable(t):
            continue
        parsed = split_tag(t)
        if not parsed:
            continue
        p, core, suf = parsed
        if p != cur_prefix:
            continue
        if suffix_shape(suf) != cur_shape:
            continue
        if len(core) != len(cur_core):
            continue
        # Some registries (linuxserver.io/lscr.io in particular) publish EXTRA tags for a
        # same-app-version base-image rebuild, shaped like the release tag plus a YYYYMMDD-ish
        # trailing component (e.g. bookstack 26.05.2 coexists with 26.05.20260608). That is not a
        # newer application release — it is a rebuild of an existing one — so any component that
        # looks like a calendar date (>= 20000, comfortably above any real release counter we saw
        # across all 58 pinned-semver images, all of which stayed under 10000) is excluded as a
        # candidate "newest" rather than misread as a huge version jump.
        if any(c >= 20000 for c in core):
            continue
        if best_core is None or core > best_core:
            best_core = core
            best = t
    if best is None:
        return None, "no matching newer tag found (same prefix/suffix shape)"
    if best_core <= cur_core:
        return None, "up to date (no tag newer than current within the same shape)"
    return (best, best_core, cur_core), None



def newest_same_major(all_tags, current_tag):
    cur = split_tag(current_tag)
    if not cur: return None
    p0, c0, s0 = cur
    best=None; bc=None
    for t in all_tags:
        if t == current_tag or is_unstable(t): continue
        pr = split_tag(t)
        if not pr: continue
        p, c, s = pr
        if p != p0 or s != s0 or len(c) != len(c0) or c[0] != c0[0]: continue
        if any(x >= 20000 for x in c): continue
        if c <= c0: continue
        if bc is None or c > bc: bc=c; best=t
    return best

def main():
    import glob, yaml
    pins = {}
    for f in sorted(glob.glob("/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/templates/*/docker-compose.yml")):
        app = f.split("/")[-2]
        d = yaml.safe_load(open(f))
        for svc, sd in (d.get("services") or {}).items():
            img = (sd or {}).get("image")
            if not img or "${" in img: continue
            repo, _, tag = img.rpartition(":") if ":" in img.split("/")[-1] else (img, "", "latest")
            pins.setdefault(img, {"repo": repo, "tag": tag, "apps": []})["apps"].append(f"{app}/{svc}")
    out = {}
    for i,(ref,p) in enumerate(sorted(pins.items())):
        e = dict(ref=ref, **p)
        host, hrepo = resolve_host_repo(p["repo"])
        if p["repo"].startswith("gitea.dooplex.hu"):
            e["status"]="internal"; out[ref]=e; continue
        try:
            tags = tags_list_all(host, hrepo)
            res, err = newest_matching(tags, p["tag"])
            e["newest_any"] = res[0] if res else None
            e["newest_same_major"] = newest_same_major(tags, p["tag"])
            e["status"] = "behind" if e["newest_any"] else "current-or-unparsed"
        except Exception as ex:
            e["status"]="error"; e["error"]=f"{type(ex).__name__}: {ex}"
        print(f"[{i+1}/{len(pins)}] {ref}: same-major={e.get('newest_same_major')} any={e.get('newest_any')} {e.get('error','')}", flush=True)
        out[ref]=e
        time.sleep(0.15)
    json.dump(out, open("02-drift.json","w"), indent=2)

if __name__ == "__main__":
    main()