324 lines
16 KiB
Python
324 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""check-published-versions.py — a released agent version must be INSTALLABLE, not merely built.
|
|
|
|
R-115. A box installs the agent from a Gitea generic package the hub vouches, never from git, and
|
|
since R-110/R-183 it also fetches the agent's sixteen config files from `raw/tag/v<version>/`.
|
|
Nothing in the build, deploy or session-end path checked that either existed, so "deployed" and
|
|
"installable" were independent states that drifted silently. **Three instances in five days:**
|
|
|
|
* R-111 (2026-07-29) 17 releases v0.97.0-v0.113.0 built and never published — a new customer
|
|
would have installed without the whole R-82 tiered-backup arc, F-CRIT-2 and F-REBOOT.
|
|
* 0.114.0 (same afternoon) built, deployed to felhom-pve, never published.
|
|
* 0.120.0 (2026-08-03) built, committed, deployed to BOTH demo hosts, never published. A
|
|
documented-path reinstall would have silently DOWNGRADED both boxes to the pre-merge
|
|
agent — and would have *succeeded* while doing it.
|
|
|
|
THE INVARIANTS — there are TWO now, and the second is R-188's price.
|
|
|
|
(1) For every `v<semver>` git tag in this repo: the matching generic package must be DOWNLOADABLE,
|
|
and the tag must serve the agent's configs.
|
|
|
|
(2) No PUBLISHED version may be missing its tag.
|
|
|
|
Invariant (2) is new (R-188, 2026-08-03) and it exists because `release-agent.sh` now pushes the tag
|
|
AFTER publishing. The old order pushed the tag first, and the old comment said why: a tag with no
|
|
package is caught here, a package with no tag is invisible, because the Gitea package LISTING api
|
|
needs a token this gate does not have. That reasoning was sound and the ordering was still wrong —
|
|
the tag push is what wakes CI, so every correct release had a ~50% chance of running this gate in the
|
|
seconds before its own package existed and mailing the operator a failure for a release that worked
|
|
(measured across two releases: runs 12/13 and 17/18, same shas, opposite results).
|
|
|
|
Moving the push does not get to trade invariant (2) away, so it is asserted here instead — WITHOUT a
|
|
token, and therefore as a BOUNDED PROBE rather than an enumeration:
|
|
|
|
* the FRONTIER — the versions immediately above the highest tag. This is the realistic failure the
|
|
new ordering makes possible: publish succeeds, tag push fails, so the orphan is exactly one
|
|
version beyond the newest tag.
|
|
* the GAPS — patch versions that fall between two existing tags and have no tag of their own.
|
|
|
|
Re-measured 2026-08-03, not assumed: `GET /api/v1/packages/admin?type=generic` answers **401** with no
|
|
token, so absence still cannot be proven. The probe set is PRINTED on every run, because a check whose
|
|
coverage is invisible reads as a guarantee it is not making.
|
|
|
|
The task's §8.4 asked for a different one — *"the version the hub tells machines to install must be
|
|
downloadable"* — and that is the better invariant in principle. **It is not implementable from CI,
|
|
and that was measured rather than assumed:** the hub's artifact manifest
|
|
(`GET /api/v1/artifacts/<customer>`) answers **401** without a per-customer retrieval passphrase,
|
|
and the Gitea package LISTING api answers **401** without a token, while the package DOWNLOAD url
|
|
and the git TAGS api are both anonymous. So a credential-free gate can ask *"is this version
|
|
installable"* but not *"which version is vouched"*. Putting an operator credential into CI to close
|
|
that gap is the operator's call, not a gate author's — it is recorded as a limitation below and as a
|
|
backlog row rather than quietly assumed away.
|
|
|
|
**What this invariant does catch: all three instances above.** `release-agent.sh` creates the tag and
|
|
publishes in one act, so a release whose publish was skipped, failed, or was forgotten leaves a tag
|
|
with no package — which is exactly what this refuses. It needs no version floor: tags begin at
|
|
v0.120.0, which is published.
|
|
|
|
**What it does NOT catch, stated plainly:** the hub vouching a version that was never released at
|
|
all (no tag, no package). Nothing here can see that; it belongs at vouch time, in the hub. → R-184.
|
|
Nor does the converse probe prove that NO untagged package exists — only that none exists at the
|
|
probed versions, which are printed. Closing that properly needs a read token in CI (→ R-184).
|
|
|
|
FAIL-CLOSED. A network error, an unparseable response or an unreachable Gitea is exit **2
|
|
INCONCLUSIVE**, naming every URL tried — never a pass. "Cannot determine" is not "fine": that is the
|
|
standing rule this project earned twice, and a gate that green-lights on its own blindness is worse
|
|
than no gate, because it looks like coverage.
|
|
|
|
Pure python3 + urllib, NO curl and no third-party module: the CI runner is a host-mode container
|
|
carrying python3 and git and nothing else, and an earlier workflow step died on
|
|
`curl: command not found`.
|
|
|
|
python3 scripts/check-published-versions.py
|
|
|
|
Exit: 0 both invariants hold · 1 either is violated · 2 could not be determined.
|
|
Env: GITEA_BASE overrides the Gitea root (CI sets the in-cluster service URL).
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
GITEA_BASE = os.environ.get("GITEA_BASE", "https://gitea.dooplex.hu").rstrip("/")
|
|
OWNER = "admin"
|
|
REPO = "felhom-agent"
|
|
PKG = "felhom-agent"
|
|
TIMEOUT = 25
|
|
|
|
# One config the installer fetches. Its presence proves the TAG's tree carries the configs the
|
|
# sixteen `fetch_raw` calls will ask for — a tag that exists but predates them would 404 a box
|
|
# mid-install, on a virgin machine, as root.
|
|
PROBE_CONFIG = "configs/felhom-agent.service"
|
|
|
|
TAG_RE = re.compile(r"^v(\d+\.\d+\.\d+)$")
|
|
|
|
# THE retention number, read from the one file that owns it. A check and the policy it enforces
|
|
# must read the same number from the same place, or they drift and the drift looks like a defect
|
|
# in something else — which is exactly what happened on 2026-08-08/09 (R-287).
|
|
RETENTION_FILE = os.environ.get("FELHOM_RETENTION_POLICY") or \
|
|
os.path.join(os.path.dirname(os.path.abspath(__file__)), "retention-policy.json") # the env: decoy tests only
|
|
|
|
|
|
def retention_kept():
|
|
"""How many of the newest generic versions the registry is expected to still serve.
|
|
|
|
Fails CLOSED and LOUD: a missing or unreadable policy file makes the check INCONCLUSIVE
|
|
rather than silently unbounded. An unbounded check would re-create the red this fixed; a
|
|
silently-bounded one would be worse.
|
|
"""
|
|
with open(RETENTION_FILE, encoding="utf-8") as fh:
|
|
n = json.load(fh)["generic_versions_kept"]
|
|
if not isinstance(n, int) or n < 1:
|
|
raise ValueError("generic_versions_kept must be a positive int, got %r" % (n,))
|
|
return n
|
|
|
|
|
|
def retention_pinned():
|
|
"""D50 = A: the versions kept besides the newest N — the hub's MinAgent floor and the day-0 agent. Copies of hub
|
|
state the vouching session writes into the policy file (CI holds no hub secret). Missing key = no pins."""
|
|
with open(RETENTION_FILE, encoding="utf-8") as fh:
|
|
pins = json.load(fh).get("pinned") or {}
|
|
if not isinstance(pins, dict):
|
|
raise ValueError("pinned must be an object, got %r" % (pins,))
|
|
out = {}
|
|
for name, v in pins.items():
|
|
if not isinstance(v, str) or not re.match(r"^\d+\.\d+\.\d+$", v):
|
|
raise ValueError("pinned %s must be X.Y.Z, got %r" % (name, v))
|
|
out[name] = v
|
|
return out
|
|
|
|
tried = []
|
|
|
|
|
|
def _get(url, want_body=False):
|
|
"""GET a URL. Returns (status, body_or_None). Network failure raises."""
|
|
tried.append(url)
|
|
req = urllib.request.Request(url, method="GET")
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
|
body = r.read() if want_body else None
|
|
return r.status, body
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, None
|
|
|
|
|
|
def inconclusive(msg):
|
|
print("INCONCLUSIVE:", msg)
|
|
print(" URLs tried (a 'no access' claim must name its attempts):")
|
|
for u in tried:
|
|
print(" ", u)
|
|
sys.exit(2)
|
|
|
|
|
|
def _pkg_exists(version):
|
|
"""True iff the generic package for `version` is downloadable anonymously."""
|
|
url = "%s/api/packages/%s/generic/%s/%s/%s" % (GITEA_BASE, OWNER, PKG, version, PKG)
|
|
status, _ = _get(url)
|
|
return status == 200, url
|
|
|
|
|
|
def untagged_probe_set(versions):
|
|
"""The versions to probe for invariant (2), as (version, why) pairs.
|
|
|
|
Bounded on purpose and printed by the caller: the package listing api needs a token (401,
|
|
re-measured 2026-08-03), so absence cannot be enumerated. What CAN be done is to probe the
|
|
places an orphan would actually land.
|
|
|
|
FRONTIER — a publish that succeeded followed by a tag push that failed leaves the orphan
|
|
exactly one version past the newest tag. This is the failure mode the R-188
|
|
reordering makes possible, so it is the one that must not be guesswork.
|
|
GAPS — a patch number skipped between two consecutive tags. Bounded per gap so a typo'd
|
|
tag (v0.130.0 after v0.121.1) cannot turn this into a thousand requests.
|
|
"""
|
|
parsed = sorted(tuple(int(p) for p in v.split(".")) for v in versions)
|
|
have = set(parsed)
|
|
out = []
|
|
if not parsed:
|
|
return out
|
|
|
|
hi = parsed[-1]
|
|
for cand, why in (
|
|
((hi[0], hi[1], hi[2] + 1), "next patch after the newest tag"),
|
|
((hi[0], hi[1], hi[2] + 2), "second patch after the newest tag"),
|
|
((hi[0], hi[1] + 1, 0), "next minor after the newest tag"),
|
|
((hi[0] + 1, 0, 0), "next major after the newest tag"),
|
|
):
|
|
if cand not in have:
|
|
out.append(("%d.%d.%d" % cand, why))
|
|
|
|
MAX_GAP_PROBES = 12
|
|
for a, b in zip(parsed, parsed[1:]):
|
|
if a[0] != b[0] or a[1] != b[1]:
|
|
continue # a minor/major step is not a patch gap
|
|
for patch in range(a[2] + 1, min(b[2], a[2] + 1 + MAX_GAP_PROBES)):
|
|
cand = (a[0], a[1], patch)
|
|
if cand not in have:
|
|
out.append(("%d.%d.%d" % cand, "patch gap between v%d.%d.%d and v%d.%d.%d" % (a + b)))
|
|
return out
|
|
|
|
|
|
def main():
|
|
print("check-published-versions — every released agent version must be INSTALLABLE")
|
|
print(" gitea:", GITEA_BASE)
|
|
|
|
tags_url = "%s/api/v1/repos/%s/%s/tags?limit=200" % (GITEA_BASE, OWNER, REPO)
|
|
try:
|
|
status, body = _get(tags_url, want_body=True)
|
|
except Exception as e:
|
|
inconclusive("cannot reach Gitea to list tags: %s" % e)
|
|
if status != 200 or not body:
|
|
inconclusive("tags api returned HTTP %s — cannot enumerate releases" % status)
|
|
try:
|
|
tags = [t["name"] for t in json.loads(body.decode("utf-8"))]
|
|
except Exception as e:
|
|
inconclusive("tags api response is not the expected JSON: %s" % e)
|
|
|
|
versions = sorted(m.group(1) for m in (TAG_RE.match(t) for t in tags) if m)
|
|
if not versions:
|
|
# Not a failure: a repo legitimately has no version tags before its first release. Say so
|
|
# loudly rather than reporting a vacuous pass — an empty check that prints OK is how an
|
|
# unexamined thing becomes a documented-clean one.
|
|
print(" no v<semver> tags in this repo yet — nothing to check, and nothing proven")
|
|
print("\ncheck-published-versions: NOTHING TO CHECK")
|
|
return 0
|
|
all_versions = versions
|
|
try:
|
|
keep = retention_kept()
|
|
pins = retention_pinned()
|
|
except Exception as e:
|
|
inconclusive("cannot read the retention policy (%s): %s" % (RETENTION_FILE, e))
|
|
|
|
# Bound the assertion to what the registry is expected to still hold. Sorted by SEMVER, not
|
|
# lexically: "0.9.0" > "0.10.0" as strings, and that would silently drop the wrong end.
|
|
def _key(v):
|
|
return tuple(int(x) for x in v.split("."))
|
|
window = sorted(all_versions, key=_key)[-keep:]
|
|
# D50 = A: the pinned versions are asserted wherever they sit. A pin with no tag cannot be installed at all —
|
|
# a failure, never a skip.
|
|
unpinned_tags = sorted(("%s=%s" % (k, v)) for k, v in pins.items() if v not in all_versions)
|
|
versions = sorted(set(window) | set(v for v in pins.values() if v in all_versions), key=_key)
|
|
dropped = [v for v in all_versions if v not in versions]
|
|
|
|
print(" %d released version(s); retention policy keeps the newest %d plus the pinned %s" % (
|
|
len(all_versions), keep, ", ".join("%s=%s" % kv for kv in sorted(pins.items())) or "(none)"))
|
|
print(" verifying: %s" % ", ".join(versions))
|
|
if dropped:
|
|
# NEVER silent. A bounded check that does not say what it stopped covering is how a
|
|
# narrowing becomes permanent by accident.
|
|
print(" NOT ASSERTED (older than the retention window, and therefore not expected to be")
|
|
print(" downloadable): %s" % ", ".join(dropped))
|
|
print(" ^ these versions still have git TAGS and are still installable in the sense that")
|
|
print(" their configs resolve; what is no longer asserted is the BINARY's presence.")
|
|
|
|
bad = []
|
|
for pin in unpinned_tags:
|
|
bad.append((pin, ["the pinned version has NO git tag — nothing can install it (D50 = A)"]))
|
|
print(" FAIL pinned %s: no git tag v%s" % (pin, pin.split("=")[1]))
|
|
for v in versions:
|
|
pkg_url = "%s/api/packages/%s/generic/%s/%s/%s" % (GITEA_BASE, OWNER, PKG, v, PKG)
|
|
raw_url = "%s/%s/%s/raw/tag/v%s/%s" % (GITEA_BASE, OWNER, REPO, v, PROBE_CONFIG)
|
|
try:
|
|
pkg_status, _ = _get(pkg_url)
|
|
raw_status, _ = _get(raw_url)
|
|
except Exception as e:
|
|
inconclusive("network failure while checking v%s: %s" % (v, e))
|
|
|
|
problems = []
|
|
if pkg_status != 200:
|
|
problems.append("binary NOT downloadable (HTTP %s at %s)" % (pkg_status, pkg_url))
|
|
if raw_status != 200:
|
|
problems.append("tag does not serve %s (HTTP %s) — a box would 404 mid-install"
|
|
% (PROBE_CONFIG, raw_status))
|
|
if problems:
|
|
bad.append((v, problems))
|
|
print(" FAIL v%s:" % v)
|
|
for p in problems:
|
|
print(" -", p)
|
|
else:
|
|
print(" ok v%s: binary downloadable + tag serves its configs" % v)
|
|
|
|
# ── invariant (2): no PUBLISHED version may be missing its tag (R-188) ──────────────────────
|
|
probes = untagged_probe_set(versions)
|
|
orphans = []
|
|
print()
|
|
print(" converse probe — a published version with no tag (bounded; the package listing api")
|
|
print(" needs a token, so this cannot enumerate). Probing %d version(s):" % len(probes))
|
|
for v, why in probes:
|
|
try:
|
|
exists, url = _pkg_exists(v)
|
|
except Exception as e:
|
|
inconclusive("network failure while probing v%s: %s" % (v, e))
|
|
mark = "PUBLISHED — NO TAG" if exists else "absent (ok)"
|
|
print(" %-10s %-42s %s" % (v, why, mark))
|
|
if exists:
|
|
orphans.append((v, url))
|
|
|
|
print()
|
|
if bad or orphans:
|
|
if orphans:
|
|
print("check-published-versions: %d PUBLISHED VERSION(S) WITH NO TAG" % len(orphans))
|
|
for v, url in orphans:
|
|
print(" v%s is downloadable at %s but has no git tag." % (v, url))
|
|
print(" A release publishes and then pushes its tag; a package with no tag means the")
|
|
print(" push failed or was skipped. The local tag is probably still in the release")
|
|
print(" clone — finish it with:")
|
|
for v, _ in orphans:
|
|
print(" git push origin v%s" % v)
|
|
print(" (and if the tag is gone, re-create it on the released commit before pushing.)")
|
|
if bad:
|
|
print("check-published-versions: %d RELEASED VERSION(S) NOT INSTALLABLE" % len(bad))
|
|
print(" A tagged version with no package is a release that was BUILT and never PUBLISHED —")
|
|
print(" the R-115 defect, three times in five days. Publish it with:")
|
|
print(" scripts/release-agent.sh <version>")
|
|
if bad or orphans:
|
|
return 1
|
|
print("check-published-versions: ALL RELEASED VERSIONS INSTALLABLE, AND NONE UNTAGGED")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|