Files
felhom-agent/configs/felhom-os-apply
T
2026-10-04 11:08:43 +02:00

487 lines
23 KiB
Python
Executable File

#!/usr/bin/python3
# felhom-os-apply — the ROOT half of the agent's operating-system update leg (`11-os-updates.md` §5.4.1).
#
# Install as /usr/local/sbin/felhom-os-apply (0755 root:root). The non-root agent invokes it via `sudo -n`
# (FELHOM_OSAPPLY alias) with EXACTLY: felhom-os-apply --plan /var/lib/felhom-agent/os/plan-<id>.json
# Nothing else on the command line is accepted. Python 3, standard library only (a JSON plan cannot be parsed
# safely in sh). Tests: configs/test_felhom_os_apply.py (a fake runner; nothing real is executed).
#
# THE TRUST MODEL. The plan is written by the agent, so a broken-into agent writes whatever plan it likes. The
# protection is therefore what this file REFUSES, not where the plan came from: no removal, no downgrade, no new
# package, no package outside the plan, only Debian origin in the fast lane, only the box's own customer guest.
# Package signatures stay Debian's: apt checks every Release file against the guest's keyring, including the
# snapshot.debian.org fallback (decision 79). Nothing here is overridable from the environment.
#
# THIS RELEASE: layer "guest", lane "fast" only. The host layer and the slow lane exist in the interface and are
# REFUSED (R3, R12) until `11` §8 steps 3 and 5 enable them.
#
# Modes (plan field "mode"):
# inventory read-only for packages: `apt-get update` in the guest, then report what is installed (with origin),
# what is pending, restart-needed and health. Installs nothing.
# apply repair first, check every refusal on an `apt-get -s` simulation of EXACTLY name=version, then
# install, clean, and report the same as inventory.
# health report health only (the agent polls it after a run).
# Output: log lines on stderr and the journal (tag felhom-os-apply); the LAST stdout line is
# OSAPPLY-REPORT <one JSON object>
# which is what the agent parses. Exit 0 = done; 2 = refused (nothing changed); 3 = failed during install.
import json
import os
import re
import stat
import subprocess
import sys
import time
PLAN_DIR = "/var/lib/felhom-agent/os"
PLAN_RE = re.compile(r"^plan-[A-Za-z0-9._-]{1,80}\.json$")
AGENT_USER = "felhom-agent"
FAST_ORIGINS = ("Debian", "Debian-Security")
# Debian package name and version grammar (Debian policy §5.6.1, §5.6.12).
NAME_RE = re.compile(r"^[a-z0-9][a-z0-9+.-]+$")
VERSION_RE = re.compile(r"^(?:[0-9]+:)?[0-9][A-Za-z0-9.+~-]*$")
SNAP_RE = re.compile(r"^[0-9]{8}T[0-9]{6}Z$")
RESERVED_VMIDS = set(range(990000, 990010)) | {9999}
DRIVES_PARENT = "/mnt/felhom-drives"
SNAPSHOT_LIST = "/etc/apt/sources.list.d/felhom-os-snapshot.list"
APT_ENV = ["env", "DEBIAN_FRONTEND=noninteractive", "APT_LISTCHANGES_FRONTEND=none", "NEEDRESTART_MODE=l", "LC_ALL=C"]
DPKG_OPTS = ["-o", "Dpkg::Options::=--force-confold", "-o", "Dpkg::Options::=--force-confdef"]
MIN_FREE = 500 * 1024 * 1024
class Refused(Exception):
def __init__(self, code, reason):
super().__init__(f"{code} {reason}")
self.code, self.reason = code, reason
class Runner:
"""Runs commands for real. Tests replace it with a fake. `guest` runs inside the container via pct exec."""
def host(self, argv, timeout=600):
p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
return p.returncode, p.stdout, p.stderr
def guest(self, vmid, argv, timeout=1800):
return self.host(["/usr/sbin/pct", "exec", str(vmid), "--"] + argv, timeout)
def read_file(self, path):
with open(path) as f:
return f.read()
def stat(self, path):
return os.lstat(path)
def agent_uid(self):
import pwd
return pwd.getpwnam(AGENT_USER).pw_uid
def log(self, line):
print(line, file=sys.stderr, flush=True)
try:
subprocess.run(["logger", "-t", "felhom-os-apply", line], timeout=10)
except Exception:
pass
class Apply:
def __init__(self, runner, plan_path):
self.r = runner
self.plan_path = plan_path
self.report = {"refused": None, "mode": None}
# ---------- checks ----------
def load_plan(self):
p = self.plan_path
d, base = os.path.dirname(p), os.path.basename(p)
if d != PLAN_DIR or not PLAN_RE.match(base) or ".." in p:
raise Refused("R1", f"the plan must be {PLAN_DIR}/plan-<id>.json, got {p!r}")
try:
st = self.r.stat(p)
except OSError as e:
raise Refused("R1", f"cannot stat the plan: {e}")
if not stat.S_ISREG(st.st_mode):
raise Refused("R1", "the plan is not a regular file (a symlink or a device is refused)")
if st.st_uid != self.r.agent_uid():
raise Refused("R1", f"the plan is not owned by {AGENT_USER}")
if st.st_size > 2 * 1024 * 1024:
raise Refused("R1", "the plan is larger than 2 MB")
try:
plan = json.loads(self.r.read_file(p))
except (OSError, ValueError) as e:
raise Refused("R1", f"the plan is not valid JSON: {e}")
if not isinstance(plan, dict):
raise Refused("R1", "the plan is not a JSON object")
return plan
def check_plan(self, plan):
mode = plan.get("mode", "apply")
if mode not in ("apply", "inventory", "health"):
raise Refused("R11", f"unknown mode {mode!r}")
if plan.get("layer") != "guest":
raise Refused("R12", f"layer {plan.get('layer')!r} is refused in this release (guest only)")
if plan.get("lane", "fast") != "fast":
raise Refused("R3", "the slow lane is refused in this release")
vmid = plan.get("vmid")
if not isinstance(vmid, int) or isinstance(vmid, bool) or vmid <= 0:
raise Refused("R11", f"vmid must be a positive integer, got {vmid!r}")
rid = plan.get("release_id", "")
if not isinstance(rid, str) or not re.match(r"^[A-Za-z0-9._:-]{1,80}$", rid):
raise Refused("R11", f"release_id {rid!r} is not a plain id")
if plan.get("allow_new"):
raise Refused("R6", "allow_new is a slow-lane field; the fast lane never adds a package")
pk = plan.get("packages", [])
if not isinstance(pk, list) or (mode == "apply" and not pk):
raise Refused("R11", "packages must be a non-empty list in apply mode")
seen = set()
for e in pk:
if not isinstance(e, dict):
raise Refused("R11", "every package entry must be an object")
n, v, o = e.get("name"), e.get("version"), e.get("origin")
if not isinstance(n, str) or not NAME_RE.match(n):
raise Refused("R11", f"package name {n!r} is not a Debian package name")
if not isinstance(v, str) or not VERSION_RE.match(v):
raise Refused("R11", f"version {v!r} of {n} is not a Debian version string")
if n in seen:
raise Refused("R11", f"package {n} is named twice")
seen.add(n)
if o not in FAST_ORIGINS:
raise Refused("R2", f"{n}: origin {o!r} is not Debian / Debian-Security (the fast lane, `11` C3)")
snap = plan.get("snapshot", "")
if snap and not SNAP_RE.match(snap):
raise Refused("R11", f"snapshot {snap!r} is not YYYYMMDDTHHMMSSZ")
return mode, vmid
def check_guest(self, vmid):
if vmid in RESERVED_VMIDS:
raise Refused("R10", f"vmid {vmid} is a reserved scratch vmid")
try:
conf = self.r.read_file(f"/etc/pve/lxc/{vmid}.conf")
except OSError:
raise Refused("R10", f"vmid {vmid} is not a container on this host")
cur = conf.split("\n[", 1)[0] # the current config, not a snapshot section
binds = [l for l in cur.splitlines() if re.match(r"^mp[0-9]+: " + re.escape(DRIVES_PARENT) + r",", l)]
if not binds:
raise Refused("R10", f"vmid {vmid} does not bind {DRIVES_PARENT} — it is not this box's customer guest")
lock = [l for l in cur.splitlines() if l.startswith("lock:")]
if lock:
raise Refused("R9", f"vmid {vmid} is locked ({lock[0].split(':', 1)[1].strip()}) — a backup or restore is running")
rc, out, _ = self.r.host(["/usr/sbin/pct", "status", str(vmid)])
if rc != 0 or "running" not in out:
raise Refused("R10", f"vmid {vmid} is not running")
# ---------- guest helpers ----------
def g(self, argv, timeout=1800):
return self.r.guest(self.vmid, argv, timeout)
def installed(self):
rc, out, _ = self.g(["dpkg-query", "-W", "-f", "${Package}\t${Version}\t${db:Status-Abbrev}\n"])
res = {}
for l in out.splitlines():
parts = l.split("\t")
if len(parts) == 3 and parts[2].startswith("ii"):
res[parts[0]] = parts[1]
return res
def dpkg_cmp(self, a, op, b):
rc, _, _ = self.g(["dpkg", "--compare-versions", a, op, b])
return rc == 0
def madison(self, name):
rc, out, _ = self.g(["apt-cache", "madison", name])
vs = set()
for l in out.splitlines():
f = [x.strip() for x in l.split("|")]
if len(f) >= 3 and f[0] == name:
vs.add(f[1])
return vs
def simulate(self, args):
rc, out, err = self.g(APT_ENV + ["apt-get", "-s", "-q"] + args)
inst, remv = [], []
for l in out.splitlines():
m = re.match(r"^Inst (\S+) (?:\[([^]]*)\] )?\((\S+) (.*?) \[[a-z0-9]+\]\)", l)
if m:
inst.append({"name": m.group(1), "from": m.group(2), "to": m.group(3), "origin": m.group(4)})
m = re.match(r"^Remv (\S+)", l)
if m:
remv.append(m.group(1))
return rc, inst, remv, out + err
@staticmethod
def origin_name(origin):
# "Debian:13.7/stable, Debian-Security:13/stable-security" -> {"Debian", "Debian-Security"}
return {o.strip().split(":")[0] for o in origin.split(",") if o.strip()}
def free_bytes(self):
rc, out, _ = self.g(["df", "-B1", "--output=avail", "/"])
try:
return int(out.strip().splitlines()[-1])
except (ValueError, IndexError):
return -1
def apt_lock_held(self):
rc, out, _ = self.g(["fuser", "/var/lib/dpkg/lock-frontend", "/var/lib/dpkg/lock"])
return rc == 0 and out.strip() != ""
def health(self):
"""The guest's signals: every container's state + health, the controller's own health, the network."""
rc, out, _ = self.g(["docker", "ps", "-a", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}"], timeout=60)
cont = {}
for l in out.splitlines():
p = l.split("\t")
if len(p) == 3:
h = "healthy" if "(healthy)" in p[2] else "unhealthy" if "(unhealthy)" in p[2] else \
"starting" if "(health: starting)" in p[2] else "none"
cont[p[0]] = {"state": p[1], "health": h}
nrc, _, _ = self.g(["getent", "hosts", "deb.debian.org"], timeout=30)
return {"docker_ok": rc == 0, "containers": cont,
"controller": cont.get("felhom-controller", {}).get("health", "absent"),
"network_ok": nrc == 0}
def restart_needed(self):
"""Processes still mapping deleted files, OUTSIDE docker containers (C11)."""
script = ('for p in /proc/[0-9]*; do grep -q "(deleted)" $p/maps 2>/dev/null || continue; '
'grep -q "docker" $p/cgroup 2>/dev/null && continue; echo "${p#/proc/} $(cat $p/comm 2>/dev/null)"; done')
rc, out, _ = self.g(["sh", "-c", script], timeout=120)
procs = sorted({l.split(" ", 1)[1] for l in out.splitlines() if " " in l})
pid1 = any(l.split(" ", 1)[0] == "1" for l in out.splitlines())
return procs, pid1
def inventory(self):
inst = self.installed()
names = sorted(inst)
origins = {}
for i in range(0, len(names), 200):
rc, out, _ = self.g(["apt-cache", "policy"] + names[i:i + 200])
cur, star = None, False
for l in out.splitlines():
if not l.startswith(" "):
cur, star = l.rstrip(":"), False
continue
s = l.strip()
if s.startswith("*** "):
star = True
continue
if star and cur and re.match(r"^[0-9-]+ ", s):
if "/var/lib/dpkg/status" in s:
origins.setdefault(cur, "local")
else:
origins[cur] = s
continue
if star and not re.match(r"^[0-9-]+ ", s):
star = False
# Map an index URL to an origin name the hub understands.
def oname(src):
if src in (None, "local"):
return "unknown"
if "security" in src and "debian" in src:
return "Debian-Security"
if "docker.com" in src:
return "Docker"
if "debian" in src:
return "Debian"
return "other"
rc, pend, remv, _ = self.simulate(["dist-upgrade"])
return {
"installed": [{"name": n, "version": inst[n], "origin": oname(origins.get(n))} for n in names],
"pending": [{"name": p["name"], "from": p["from"], "to": p["to"],
"origin": sorted(self.origin_name(p["origin"]))} for p in pend],
}
# ---------- the run ----------
def run(self):
plan = self.load_plan()
self.mode, self.vmid = self.check_plan(plan)
self.report.update(mode=self.mode, release_id=plan.get("release_id"), vmid=self.vmid)
self.check_guest(self.vmid)
log = self.r.log
if self.mode == "health":
self.report["health"] = self.health()
return 0
log(f"os-apply: START release={plan.get('release_id')} layer=guest:{self.vmid} lane=fast mode={self.mode} packages={len(plan.get('packages', []))}")
if self.apt_lock_held():
raise Refused("R9", "another apt/dpkg holds the lock in the guest")
self.report["health_before"] = self.health()
if self.mode == "apply":
self.repair()
rc, out, err = self.g(APT_ENV + ["apt-get", "-q", "update"], timeout=600)
if rc != 0:
raise Refused("R7", f"apt-get update failed in the guest: {(out + err).strip().splitlines()[-1:]}")
if self.mode == "apply":
rc = self.apply(plan)
if rc:
return rc
procs, pid1 = self.restart_needed()
self.report.update(self.inventory())
self.report["restart_needed"] = procs
self.report["docker_restart_needed"] = any(p in ("dockerd", "containerd") for p in procs)
self.report["reboot_needed"] = pid1
self.report["health_after"] = self.health()
return 0
def repair(self):
rc, before, _ = self.g(["dpkg", "--audit"])
self.g(APT_ENV + ["dpkg", "--configure", "-a", "--force-confold"])
rc2, out, err = self.g(APT_ENV + ["apt-get", "-f", "install", "-y", "-q"] + DPKG_OPTS)
_, after, _ = self.g(["dpkg", "--audit"])
configured = len([l for l in before.splitlines() if l.startswith(" ")])
fixed = len(re.findall(r"^Setting up ", out, re.M))
self.report["repair"] = {"half_configured_before": configured, "fixed": fixed, "clean_after": after.strip() == ""}
self.r.log(f"os-apply: REPAIR configured={configured} fixed={fixed}")
if after.strip():
raise Refused("R13", "dpkg is still broken after the repair: " + after.strip().splitlines()[0])
def apply(self, plan):
log = self.r.log
inst = self.installed()
upgrade, already, notinst = [], 0, 0
for e in plan["packages"]:
n, v = e["name"], e["version"]
if n not in inst:
notinst += 1
continue
if not self.dpkg_cmp(v, "gt", inst[n]):
already += 1
continue
upgrade.append((n, v))
from_snap = 0
missing = [(n, v) for n, v in upgrade if v not in self.madison(n)]
if missing:
snap = plan.get("snapshot", "")
if not snap:
raise Refused("R7", f"{missing[0][0]}={missing[0][1]} is not downloadable and the plan names no snapshot")
self.add_snapshot_sources(snap)
still = [(n, v) for n, v in missing if v not in self.madison(n)]
if still:
self.remove_snapshot_sources()
raise Refused("R7", f"{still[0][0]}={still[0][1]} is not downloadable, not even from snapshot {snap}")
from_snap = len(missing)
try:
log(f"os-apply: PLAN upgrade={len(upgrade)} already={already} not-installed={notinst} from-snapshot={from_snap}")
self.report["plan"] = {"upgrade": len(upgrade), "already": already, "not_installed": notinst, "from_snapshot": from_snap}
if not upgrade:
self.report["upgraded"] = []
log("os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)")
return 0
args = ["install", "--only-upgrade", "--no-install-recommends"] + [f"{n}={v}" for n, v in upgrade]
rc, sim, remv, text = self.simulate(args)
if rc != 0:
tail = text.strip().splitlines()[-1] if text.strip() else ""
raise Refused("R7", "the simulation failed: " + tail)
if remv:
raise Refused("R4", f"the plan would remove {', '.join(remv[:5])}")
want = dict(upgrade)
for p in sim:
if p["from"] is None:
raise Refused("R6", f"the plan would add a package that is not installed: {p['name']}")
if p["name"] not in want:
raise Refused("R6", f"the plan would touch {p['name']}, which is not in the plan")
if p["to"] != want[p["name"]]:
raise Refused("R6", f"{p['name']} would go to {p['to']}, not the approved {want[p['name']]}")
if not self.dpkg_cmp(p["to"], "gt", p["from"]):
raise Refused("R5", f"{p['name']} would be downgraded {p['from']} -> {p['to']}")
if not self.origin_name(p["origin"]) & set(FAST_ORIGINS):
raise Refused("R2", f"{p['name']} would come from {p['origin']}, not Debian")
need = self.download_bytes(args)
free = self.free_bytes()
if free >= 0 and free < max(MIN_FREE, 3 * need):
raise Refused("R8", f"free space {free} B is below max(500 MB, 3 x download {need} B)")
t0 = time.time()
rc, out, err = self.g(APT_ENV + ["apt-get", "-y", "-q"] + DPKG_OPTS + args)
secs = time.time() - t0
# dpkg says "Installing new version of config file X" when X was NOT changed locally (the package's new
# version is taken), and "Configuration file 'X'" + "Keeping old config file" when it was (--force-confold
# keeps the local one; the package's version lands as X.dpkg-dist). Measured live 2026-10-04 (debian_version).
conflict = None
for l in (out + err).splitlines():
m = re.search(r"Installing new version of config file (\S+?)\s*\.\.\.", l)
if m:
log(f"os-apply: CONFFILE updated {m.group(1)} (it was not changed locally)")
m = re.search(r"Configuration file '([^']+)'", l)
if m:
conflict = m.group(1)
if conflict and "Keeping old config file" in l:
log(f"os-apply: CONFFILE kept {conflict} (changed locally; the package's version is {conflict}.dpkg-dist)")
self.report.setdefault("conffiles_kept", []).append(conflict)
conflict = None
self.g(["apt-get", "clean"])
if rc != 0:
_, aud, _ = self.g(["dpkg", "--audit"])
first = aud.strip().splitlines()[0] if aud.strip() else "clean"
log(f"os-apply: FAILED rc={rc} step=install — dpkg state: {first}")
self.report["failed"] = {"rc": rc, "dpkg_audit": first, "tail": (out + err).strip().splitlines()[-3:]}
return 3
self.report["upgraded"] = [{"name": n, "version": v} for n, v in upgrade]
self.report["seconds"] = round(secs, 1)
log(f"os-apply: DONE rc=0 seconds={secs:.1f} upgraded={len(upgrade)}")
return 0
finally:
if from_snap:
self.remove_snapshot_sources()
def download_bytes(self, args):
rc, out, _ = self.g(APT_ENV + ["apt-get", "-s", "-o", "Debug::NoLocking=1", "--print-uris", "-q"] + args)
total = 0
for l in out.splitlines():
m = re.match(r"^'[^']+' \S+ ([0-9]+) ", l)
if m:
total += int(m.group(1))
return total
def add_snapshot_sources(self, snap):
rc, out, _ = self.g(["sh", "-c", ". /etc/os-release && echo $VERSION_CODENAME"])
code = out.strip()
if not re.match(r"^[a-z]+$", code):
raise Refused("R7", f"cannot read the guest's Debian codename ({code!r})")
body = (f"deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/{snap} {code} main\n"
f"deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/{snap} {code}-security main\n")
self.r.guest_write(self.vmid, SNAPSHOT_LIST, body)
self.r.log(f"os-apply: SNAPSHOT using snapshot.debian.org/{snap} for versions no longer published (decision 79)")
rc, out, err = self.g(APT_ENV + ["apt-get", "-q", "update"], timeout=600)
if rc != 0:
self.remove_snapshot_sources()
raise Refused("R7", "apt-get update against snapshot.debian.org failed")
def remove_snapshot_sources(self):
self.g(["rm", "-f", SNAPSHOT_LIST])
self.g(APT_ENV + ["apt-get", "-q", "update"], timeout=600)
def guest_write(self, vmid, path, body):
"""Write a small text file inside the guest via `pct exec … tee` (stdin), never via a shell string."""
p = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", path], input=body,
capture_output=True, text=True, timeout=60)
if p.returncode != 0:
raise Refused("R7", f"could not write {path} in the guest")
Runner.guest_write = guest_write
def main(argv, runner=None):
r = runner or Runner()
if len(argv) != 3 or argv[1] != "--plan":
r.log("os-apply: REFUSED: R1 usage: felhom-os-apply --plan /var/lib/felhom-agent/os/plan-<id>.json")
print("OSAPPLY-REPORT " + json.dumps({"refused": {"code": "R1", "reason": "usage"}}))
return 2
a = Apply(r, argv[2])
try:
rc = a.run()
except Refused as e:
r.log(f"os-apply: REFUSED: {e.code} {e.reason}")
a.report["refused"] = {"code": e.code, "reason": e.reason}
rc = 2
except subprocess.TimeoutExpired as e:
r.log(f"os-apply: FAILED rc=124 step=timeout — {e.cmd}")
a.report["failed"] = {"rc": 124, "timeout": str(e.cmd)[:200]}
rc = 3
print("OSAPPLY-REPORT " + json.dumps(a.report, sort_keys=True))
return rc
if __name__ == "__main__":
if os.geteuid() != 0:
print("felhom-os-apply: must run as root (via sudo)", file=sys.stderr)
sys.exit(2)
sys.exit(main(sys.argv))