Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)
gates / gates (push) Successful in 19s
gates / gates (push) Successful in 19s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
# /etc/felhom/crash-guard.conf — read by /usr/local/sbin/felhom-crash-guard (`11` §5.9).
|
||||
# The LIMIT-th unclean stop within WINDOW_MINUTES leaves the box off. Decided by CC unattended — operator may reverse.
|
||||
LIMIT=3
|
||||
WINDOW_MINUTES=60
|
||||
# kernel.panic while armed: seconds after a crash before the kernel restarts the box.
|
||||
PANIC_SECONDS=10
|
||||
# A tripped guard re-arms after this many hours of normal running (or `felhom-crash-guard rearm`).
|
||||
REARM_HOURS=24
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/python3
|
||||
# felhom-crash-guard — a crashed host restarts by itself, but not forever (`09` decision 88, R-851, `11` §5.9).
|
||||
#
|
||||
# Install as /usr/local/sbin/felhom-crash-guard (0755 root:root), with felhom-crash-guard.service (boot / clean-stop)
|
||||
# and felhom-crash-guard-check.timer (hourly re-arm check). Python 3, standard library only.
|
||||
# Tests: configs/test_felhom_crash_guard.py (temp dirs; nothing real is touched).
|
||||
#
|
||||
# WHAT IT DOES
|
||||
# boot early at every boot. Was the previous boot ended CLEANLY? (the clean-stop marker exists). If not, this
|
||||
# boot follows an UNCLEAN stop — a kernel crash, a power cut or a hard reset (they cannot be told apart
|
||||
# on these boxes: measured 2026-10-04 on demo-hp, efi_pstore is on yet saved NOTHING for a real panic;
|
||||
# the journal and `last` show only "no shutdown"). It records the unclean boot, counts those in the last
|
||||
# WINDOW_MINUTES, and sets kernel.panic:
|
||||
# - fewer than LIMIT-1 recent unclean boots → kernel.panic = PANIC_SECONDS (a crash restarts the box);
|
||||
# - LIMIT-1 or more → the guard TRIPS: kernel.panic = 0, so the LIMIT-th crash within the window
|
||||
# leaves the box OFF (operator's own words: "if it crashes 3 times within one hour, it stays off").
|
||||
# A tripped guard stays tripped across further boots until it re-arms.
|
||||
# clean-stop ExecStop of the service: writes the clean-stop marker during an orderly shutdown or reboot.
|
||||
# check hourly: a tripped guard re-arms after REARM_HOURS of normal running (since the trip AND since boot).
|
||||
# rearm the operator re-arms by hand (`felhom-crash-guard rearm`).
|
||||
# status prints the state.
|
||||
# The state is /var/lib/felhom-crash-guard/state.json (0644: the non-root agent reads it into its host report).
|
||||
# Before the service runs (very early boot) the kernel default kernel.panic = 0 applies, so a crash THAT early leaves
|
||||
# the box off — the safe side: a box that cannot reach userspace must not loop.
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
CONF = "/etc/felhom/crash-guard.conf"
|
||||
STATE_DIR = "/var/lib/felhom-crash-guard"
|
||||
DEFAULTS = {"LIMIT": 3, "WINDOW_MINUTES": 60, "PANIC_SECONDS": 10, "REARM_HOURS": 24}
|
||||
|
||||
|
||||
class Env:
|
||||
"""Paths and clock; tests replace them."""
|
||||
|
||||
def __init__(self, conf=CONF, state_dir=STATE_DIR, panic_path="/proc/sys/kernel/panic",
|
||||
uptime_path="/proc/uptime", boot_id_path="/proc/sys/kernel/random/boot_id"):
|
||||
self.conf, self.state_dir = conf, state_dir
|
||||
self.panic_path, self.uptime_path, self.boot_id_path = panic_path, uptime_path, boot_id_path
|
||||
|
||||
def now(self):
|
||||
return time.time()
|
||||
|
||||
def log(self, line):
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
try:
|
||||
import subprocess
|
||||
subprocess.run(["logger", "-t", "felhom-crash-guard", line], timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def iso(t):
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t))
|
||||
|
||||
|
||||
def parse_iso(s):
|
||||
import calendar
|
||||
return calendar.timegm(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))
|
||||
|
||||
|
||||
def load_conf(env):
|
||||
c = dict(DEFAULTS)
|
||||
try:
|
||||
for line in open(env.conf):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = (x.strip() for x in line.split("=", 1))
|
||||
if k in c and v.isdigit() and int(v) >= (1 if k != "PANIC_SECONDS" else 1):
|
||||
c[k] = int(v)
|
||||
except OSError:
|
||||
pass
|
||||
return c
|
||||
|
||||
|
||||
def state_path(env):
|
||||
return os.path.join(env.state_dir, "state.json")
|
||||
|
||||
|
||||
def marker_path(env):
|
||||
return os.path.join(env.state_dir, "clean-stop")
|
||||
|
||||
|
||||
def load_state(env):
|
||||
try:
|
||||
with open(state_path(env)) as f:
|
||||
s = json.load(f)
|
||||
return s if isinstance(s, dict) else None
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def save_state(env, s):
|
||||
os.makedirs(env.state_dir, mode=0o755, exist_ok=True)
|
||||
tmp = state_path(env) + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(s, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, state_path(env))
|
||||
|
||||
|
||||
def set_panic(env, seconds):
|
||||
with open(env.panic_path, "w") as f:
|
||||
f.write(f"{seconds}\n")
|
||||
|
||||
|
||||
def read(path, default=""):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return f.read().strip()
|
||||
except OSError:
|
||||
return default
|
||||
|
||||
|
||||
def summarize(s, c, now):
|
||||
window = c["WINDOW_MINUTES"] * 60
|
||||
times = [parse_iso(t) for t in s.get("unclean_boots", [])]
|
||||
after = parse_iso(s["rearmed_at"]) if s.get("rearmed_at") else 0
|
||||
# a re-arm starts a fresh window (or the next unclean boot would trip again at once); the history stays
|
||||
s["unclean_boots_in_window"] = sum(1 for t in times if now - t <= window and t > after)
|
||||
s["unclean_boots_24h"] = sum(1 for t in times if now - t <= 86400)
|
||||
s["config"] = c
|
||||
s["updated_at"] = iso(now)
|
||||
|
||||
|
||||
def boot(env):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
try:
|
||||
up = float(read(env.uptime_path, "0").split()[0])
|
||||
except (ValueError, IndexError):
|
||||
up = 0.0
|
||||
boot_at = now - up
|
||||
prev = load_state(env)
|
||||
first = prev is None
|
||||
s = prev or {"version": 1, "unclean_boots": [], "tripped": False}
|
||||
clean = os.path.exists(marker_path(env))
|
||||
unclean = (not first) and (not clean)
|
||||
try:
|
||||
os.remove(marker_path(env))
|
||||
except OSError:
|
||||
pass
|
||||
# keep 7 days of history (the 24 h figure and the operator's view), drop older
|
||||
s["unclean_boots"] = [t for t in s.get("unclean_boots", []) if now - parse_iso(t) <= 7 * 86400]
|
||||
if unclean:
|
||||
s["unclean_boots"].append(iso(boot_at))
|
||||
s["last_boot_at"] = iso(boot_at)
|
||||
s["last_boot_unclean"] = unclean
|
||||
s["boot_id"] = read(env.boot_id_path, "unknown")
|
||||
summarize(s, c, now)
|
||||
if not s.get("tripped") and s["unclean_boots_in_window"] >= c["LIMIT"] - 1:
|
||||
s["tripped"], s["tripped_at"] = True, iso(now)
|
||||
s["tripped_reason"] = (f"{s['unclean_boots_in_window']} unclean boots within {c['WINDOW_MINUTES']} minutes — "
|
||||
f"the next crash leaves the box off (limit {c['LIMIT']})")
|
||||
env.log(f"crash-guard: TRIPPED: {s['tripped_reason']}")
|
||||
panic = 0 if s.get("tripped") else c["PANIC_SECONDS"]
|
||||
set_panic(env, panic)
|
||||
s["kernel_panic"] = panic
|
||||
s["armed"] = not s.get("tripped")
|
||||
save_state(env, s)
|
||||
env.log(f"crash-guard: boot first={first} unclean={unclean} in-window={s['unclean_boots_in_window']} "
|
||||
f"tripped={s.get('tripped')} kernel.panic={panic}")
|
||||
return 0
|
||||
|
||||
|
||||
def clean_stop(env):
|
||||
os.makedirs(env.state_dir, mode=0o755, exist_ok=True)
|
||||
with open(marker_path(env), "w") as f:
|
||||
f.write(iso(env.now()) + "\n")
|
||||
env.log("crash-guard: clean stop recorded")
|
||||
return 0
|
||||
|
||||
|
||||
def rearm(env, by):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
s = load_state(env) or {"version": 1, "unclean_boots": []}
|
||||
was = bool(s.get("tripped"))
|
||||
s["tripped"] = False
|
||||
s["armed"] = True
|
||||
s["rearmed_at"], s["rearmed_by"] = iso(now), by
|
||||
if was:
|
||||
s["last_trip"] = {"at": s.get("tripped_at"), "reason": s.get("tripped_reason")}
|
||||
s.pop("tripped_at", None)
|
||||
s.pop("tripped_reason", None)
|
||||
summarize(s, c, now)
|
||||
set_panic(env, c["PANIC_SECONDS"])
|
||||
s["kernel_panic"] = c["PANIC_SECONDS"]
|
||||
save_state(env, s)
|
||||
env.log(f"crash-guard: RE-ARMED by {by} (was tripped: {was}); kernel.panic={c['PANIC_SECONDS']}")
|
||||
return 0
|
||||
|
||||
|
||||
def check(env):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
s = load_state(env)
|
||||
if not s:
|
||||
return 0
|
||||
if s.get("tripped"):
|
||||
since = max(parse_iso(s["tripped_at"]), parse_iso(s.get("last_boot_at", s["tripped_at"])))
|
||||
if now - since >= c["REARM_HOURS"] * 3600:
|
||||
return rearm(env, f"timer ({c['REARM_HOURS']} h of normal running)")
|
||||
summarize(s, c, now)
|
||||
save_state(env, s)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv, env=None):
|
||||
env = env or Env()
|
||||
cmd = argv[1] if len(argv) == 2 else ""
|
||||
if cmd == "boot":
|
||||
return boot(env)
|
||||
if cmd == "clean-stop":
|
||||
return clean_stop(env)
|
||||
if cmd == "check":
|
||||
return check(env)
|
||||
if cmd == "rearm":
|
||||
return rearm(env, "operator")
|
||||
if cmd == "status":
|
||||
print(json.dumps(load_state(env), indent=2, sort_keys=True))
|
||||
return 0
|
||||
print("usage: felhom-crash-guard boot|clean-stop|check|rearm|status", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.geteuid() != 0:
|
||||
print("felhom-crash-guard: must run as root", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -0,0 +1,7 @@
|
||||
# Hourly: a tripped crash guard re-arms after REARM_HOURS of normal running (felhom-crash-guard check).
|
||||
[Unit]
|
||||
Description=Felhom crash guard re-arm check
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-crash-guard check
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Felhom crash guard re-arm check (hourly)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15min
|
||||
OnUnitActiveSec=1h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-crash-guard — a crashed host restarts by itself, with a limit (`09` decision 88, R-851, `11` §5.9).
|
||||
# Starts early at boot (sets kernel.panic for THIS boot); its ExecStop writes the clean-stop marker during an orderly
|
||||
# shutdown or reboot. A boot that finds no marker followed a crash, a power cut or a hard reset.
|
||||
[Unit]
|
||||
Description=Felhom crash guard (restart after a kernel crash, with a limit)
|
||||
DefaultDependencies=no
|
||||
After=local-fs.target
|
||||
Before=sysinit.target shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
RequiresMountsFor=/var/lib
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/felhom-crash-guard boot
|
||||
ExecStop=/usr/local/sbin/felhom-crash-guard clean-stop
|
||||
|
||||
[Install]
|
||||
WantedBy=sysinit.target
|
||||
+339
-27
@@ -14,7 +14,13 @@
|
||||
# file, including the snapshot.debian.org fallback (decision 79). Nothing here is overridable from the environment.
|
||||
#
|
||||
# LAYERS (agent v0.141.0): "guest" (the customer LXC, entered with `pct exec`) and "host" (this Proxmox host, run
|
||||
# directly). LANE: "fast" only — the slow lane (kernel, Proxmox, Docker) is REFUSED (R3, R14) until `11` §8 steps 5–6.
|
||||
# directly). LANE: "fast" for those two. Agent v0.142.0 adds the layer "docker" (the guest's Docker engine set, `11`
|
||||
# §5.8), which is the SLOW lane: lane "slow" only, the six Docker packages only, origin "Docker CE" only, and only
|
||||
# with an authority this file checks ITSELF (R3): a signed operator job verified with `ssh-keygen -Y verify` against
|
||||
# the ROOT-OWNED signers file (TRUST_SIGNERS), bound to this host (TRUST_FILE host_id), unexpired and never replayed;
|
||||
# or, for an unsigned ring-0 step, the root-owned TRUST_FILE saying `"ring0_slow_lane": true` (set by hand on the demo
|
||||
# boxes only). The agent's own config is NOT trusted for either: the agent can write it. A Docker step also needs
|
||||
# `live-restore` ON (R15) — without it every container restarts.
|
||||
#
|
||||
# Modes (plan field "mode"):
|
||||
# inventory `apt-get update`, then report what is installed (with origin), what is pending, and health.
|
||||
@@ -22,6 +28,10 @@
|
||||
# pending Debian / Debian-Security upgrade, for ring 0), check every refusal on an `apt-get -s`
|
||||
# simulation of EXACTLY name=version, install, clean, scan for restart-needed, report as inventory.
|
||||
# health report health only (the agent polls it after a run).
|
||||
# facts (v0.142.0) read-only versions for the hub's System page: host Debian, running and next-boot kernel,
|
||||
# held packages, kernel taint, the crash guard; guest Debian, Docker engine, containerd, live-restore.
|
||||
# live-restore-on (v0.142.0, layer guest) the ONE-TIME step of `09` decision 87: merge `"live-restore": true`
|
||||
# into the guest's /etc/docker/daemon.json and `systemctl reload docker`. NEVER a restart (R-835).
|
||||
# Output: log lines on stderr and the journal (tag felhom-os-apply); the LAST stdout line is
|
||||
# OSAPPLY-REPORT <one JSON object>
|
||||
# which is what the agent parses. Exit 0 = done; 2 = refused (nothing changed); 3 = failed during install.
|
||||
@@ -30,6 +40,7 @@
|
||||
# package for version comparisons — 272 packages ≈ 4 minutes. Versions are now compared with the HOST's dpkg (the same
|
||||
# Debian algorithm), madison/policy run once per pass for all packages, the restart scan runs only after an install,
|
||||
# and one wrapper call does the whole pass (no separate inventory call before an apply).
|
||||
import calendar
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -64,6 +75,18 @@ HOST_SLOW_RE = re.compile(r"^(linux-(image|headers|kbuild|modules|base)|proxmox-
|
||||
# after a libc6 update). Pinned by test_restart_skip_patterns_against_real_cgroups.
|
||||
RESTART_SKIP_CGROUP = {"guest": "docker", "host": ":/lxc/"}
|
||||
HOST_SERVICES = ["pveproxy", "pvedaemon", "pvestatd", "pve-cluster", "felhom-agent"]
|
||||
# The Docker engine set (`11` §5.8): the only names the docker layer may touch, from the only origin it may use.
|
||||
DOCKER_NAMES = ("containerd.io", "docker-buildx-plugin", "docker-ce", "docker-ce-cli", "docker-ce-rootless-extras",
|
||||
"docker-compose-plugin")
|
||||
DOCKER_ORIGIN = "Docker CE"
|
||||
# ROOT-OWNED trust anchors (the installer writes them; the demo boxes got them by hand, R-840). Never the agent's config.
|
||||
TRUST_FILE = "/etc/felhom/os-trust.json" # {"host_id": "...", "ring0_slow_lane": false}
|
||||
TRUST_SIGNERS = "/etc/felhom/operator-signers" # ssh allowed_signers: <key_id> namespaces="felhom-op-v1" <key>
|
||||
SIG_NAMESPACE = "felhom-op-v1"
|
||||
SIGNED_OP = "os_docker_step"
|
||||
NONCE_FILE = "/var/lib/felhom-os-apply/nonces.json"
|
||||
DAEMON_JSON = "/etc/docker/daemon.json"
|
||||
CRASH_GUARD_STATE = "/var/lib/felhom-crash-guard/state.json"
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
@@ -103,6 +126,38 @@ class Runner:
|
||||
if rc != 0:
|
||||
raise Refused("R7", f"could not write {path} in the guest")
|
||||
|
||||
def now(self):
|
||||
return time.time()
|
||||
|
||||
def sleep(self, s):
|
||||
time.sleep(s)
|
||||
|
||||
def verify_sig(self, signers, key_id, namespace, blob, sig):
|
||||
"""`ssh-keygen -Y verify` over the EXACT signed bytes. Files in a root-only temp dir; nothing via a shell."""
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory(prefix="felhom-os-apply-") as d:
|
||||
sp = os.path.join(d, "sig")
|
||||
with open(sp, "w") as f:
|
||||
f.write(sig)
|
||||
p = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", signers, "-I", key_id, "-n", namespace, "-s", sp],
|
||||
input=blob, capture_output=True, timeout=30)
|
||||
return p.returncode
|
||||
|
||||
def read_nonces(self):
|
||||
try:
|
||||
with open(NONCE_FILE) as f:
|
||||
d = json.load(f)
|
||||
return d if isinstance(d, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
def write_nonces(self, d):
|
||||
os.makedirs(os.path.dirname(NONCE_FILE), mode=0o700, exist_ok=True)
|
||||
tmp = NONCE_FILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(d, f)
|
||||
os.replace(tmp, NONCE_FILE)
|
||||
|
||||
def log(self, line):
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
try:
|
||||
@@ -143,13 +198,22 @@ class Apply:
|
||||
|
||||
def check_plan(self, plan):
|
||||
mode = plan.get("mode", "apply")
|
||||
if mode not in ("apply", "inventory", "health"):
|
||||
if mode not in ("apply", "inventory", "health", "facts", "live-restore-on"):
|
||||
raise Refused("R11", f"unknown mode {mode!r}")
|
||||
layer = plan.get("layer")
|
||||
if layer not in ("guest", "host"):
|
||||
raise Refused("R12", f"layer {layer!r} is not guest or host")
|
||||
if plan.get("lane", "fast") != "fast":
|
||||
raise Refused("R3", "the slow lane is refused in this release")
|
||||
if layer not in ("guest", "host", "docker"):
|
||||
raise Refused("R12", f"layer {layer!r} is not guest, host or docker")
|
||||
lane = plan.get("lane", "fast")
|
||||
if layer == "docker" and lane != "slow":
|
||||
raise Refused("R3", "the Docker engine is the slow lane (`11` §5.8); a fast-lane Docker plan is refused")
|
||||
if layer != "docker" and lane != "fast":
|
||||
raise Refused("R3", f"the {layer} layer has no slow lane in this release (kernel, Proxmox: `11` §8 step 6)")
|
||||
if mode == "facts" and layer != "host":
|
||||
raise Refused("R11", "facts is a host-layer mode (it reads the host and the guest)")
|
||||
if mode == "live-restore-on" and layer != "guest":
|
||||
raise Refused("R11", "live-restore-on is a guest-layer mode")
|
||||
if plan.get("undo") and layer != "docker":
|
||||
raise Refused("R5", "an undo (downgrade) exists only for the Docker layer, inside a signed job")
|
||||
vmid = plan.get("vmid")
|
||||
if not isinstance(vmid, int) or isinstance(vmid, bool) or vmid <= 0:
|
||||
raise Refused("R11", f"vmid must be a positive integer, got {vmid!r}")
|
||||
@@ -159,15 +223,18 @@ class Apply:
|
||||
if plan.get("allow_new"):
|
||||
raise Refused("R6", "allow_new is a slow-lane field; the fast lane never adds a package")
|
||||
select = plan.get("select", "listed")
|
||||
if select not in ("listed", "pending-fast"):
|
||||
if select not in ("listed", "pending-fast", "pending-docker"):
|
||||
raise Refused("R11", f"unknown select {select!r}")
|
||||
if (select == "pending-docker") != (layer == "docker" and select != "listed"):
|
||||
if select == "pending-docker" or layer == "docker":
|
||||
raise Refused("R11", f"select {select!r} does not fit layer {layer!r}")
|
||||
pk = plan.get("packages", [])
|
||||
if not isinstance(pk, list):
|
||||
raise Refused("R11", "packages must be a list")
|
||||
if mode == "apply" and select == "listed" and not pk:
|
||||
raise Refused("R11", "packages must be a non-empty list in apply mode (select listed)")
|
||||
if select == "pending-fast" and pk:
|
||||
raise Refused("R11", "select pending-fast takes no package list")
|
||||
if select in ("pending-fast", "pending-docker") and pk:
|
||||
raise Refused("R11", f"select {select} takes no package list")
|
||||
seen = set()
|
||||
for e in pk:
|
||||
if not isinstance(e, dict):
|
||||
@@ -180,6 +247,12 @@ class Apply:
|
||||
if n in seen:
|
||||
raise Refused("R11", f"package {n} is named twice")
|
||||
seen.add(n)
|
||||
if layer == "docker":
|
||||
if n not in DOCKER_NAMES or o != DOCKER_ORIGIN:
|
||||
raise Refused("R2", f"{n} ({o!r}) is not one of the six Docker packages from {DOCKER_ORIGIN!r}")
|
||||
continue
|
||||
if n in DOCKER_NAMES:
|
||||
raise Refused("R2", f"{n} is a Docker package — the slow lane (`11` §5.8), never in a {layer} plan")
|
||||
if o not in FAST_ORIGINS:
|
||||
raise Refused("R2", f"{n}: origin {o!r} is not Debian / Debian-Security (the fast lane, `11` C3)")
|
||||
if layer == "host" and HOST_SLOW_RE.match(n):
|
||||
@@ -204,6 +277,208 @@ class Apply:
|
||||
if mode != "appliance":
|
||||
raise Refused("R12", f"this box was installed as {mode!r}, not appliance — its host belongs to its owner")
|
||||
|
||||
def load_trust(self):
|
||||
"""The ROOT-OWNED trust record. Absent or agent-writable → no slow-lane authority at all (R3)."""
|
||||
try:
|
||||
st = self.r.stat(TRUST_FILE)
|
||||
except OSError:
|
||||
raise Refused("R3", f"no {TRUST_FILE} — this box has no slow-lane trust anchor")
|
||||
if st.st_uid != 0 or (st.st_mode & 0o022):
|
||||
raise Refused("R3", f"{TRUST_FILE} is not root-owned and root-only-writable — it proves nothing")
|
||||
try:
|
||||
t = json.loads(self.r.read_file(TRUST_FILE))
|
||||
except (OSError, ValueError):
|
||||
raise Refused("R3", f"{TRUST_FILE} is unreadable")
|
||||
if not isinstance(t, dict) or not isinstance(t.get("host_id"), str) or not t["host_id"]:
|
||||
raise Refused("R3", f"{TRUST_FILE} names no host_id")
|
||||
return t
|
||||
|
||||
def verify_signed(self, signed, trust):
|
||||
"""R3: an operator-signed os_docker_step, checked HERE (not by the agent): signature against the root-owned
|
||||
signers file, op, host binding, time window, and a root-owned nonce record (no replay)."""
|
||||
import base64
|
||||
if not isinstance(signed, dict) or not isinstance(signed.get("blob_b64"), str) or not isinstance(signed.get("sig"), str):
|
||||
raise Refused("R3", "the signed job is malformed")
|
||||
try:
|
||||
blob = base64.b64decode(signed["blob_b64"], validate=True)
|
||||
op = json.loads(blob)
|
||||
except (ValueError, TypeError):
|
||||
raise Refused("R3", "the signed blob is not base64 JSON")
|
||||
key_id = op.get("key_id", "")
|
||||
if not isinstance(key_id, str) or not re.match(r"^[A-Za-z0-9._-]{1,64}$", key_id):
|
||||
raise Refused("R3", "the signed blob names no plain key_id")
|
||||
try:
|
||||
st = self.r.stat(TRUST_SIGNERS)
|
||||
except OSError:
|
||||
raise Refused("R3", f"no {TRUST_SIGNERS} — no operator key to check a signed job against")
|
||||
if st.st_uid != 0 or (st.st_mode & 0o022):
|
||||
raise Refused("R3", f"{TRUST_SIGNERS} is not root-owned and root-only-writable")
|
||||
rc = self.r.verify_sig(TRUST_SIGNERS, key_id, SIG_NAMESPACE, blob, signed["sig"])
|
||||
if rc != 0:
|
||||
raise Refused("R3", f"the operator signature does not verify (ssh-keygen rc={rc})")
|
||||
if op.get("op") != SIGNED_OP:
|
||||
raise Refused("R3", f"the signed op is {op.get('op')!r}, not {SIGNED_OP}")
|
||||
if (op.get("target") or {}).get("host_id") != trust["host_id"]:
|
||||
raise Refused("R3", "the signed job is for another host")
|
||||
now = self.r.now()
|
||||
try:
|
||||
exp = calendar.timegm(time.strptime(op["expires_at"], "%Y-%m-%dT%H:%M:%SZ"))
|
||||
iss = calendar.timegm(time.strptime(op["issued_at"], "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except (KeyError, ValueError, TypeError):
|
||||
raise Refused("R3", "the signed job has no readable time window")
|
||||
if now > exp or now < iss - 120:
|
||||
raise Refused("R3", "the signed job is expired or not yet valid")
|
||||
nonce = op.get("nonce")
|
||||
if not isinstance(nonce, str) or not nonce:
|
||||
raise Refused("R3", "the signed job has no nonce")
|
||||
seen = self.r.read_nonces()
|
||||
if nonce in seen:
|
||||
raise Refused("R3", "the signed job was already used (replay)")
|
||||
seen[nonce] = exp
|
||||
self.r.write_nonces({k: v for k, v in seen.items() if v > now})
|
||||
return op.get("params") or {}
|
||||
|
||||
def docker_authority(self, plan):
|
||||
"""R3 for the docker layer: returns (who, undo). A signed job binds the EXACT package list and the undo flag."""
|
||||
trust = self.load_trust()
|
||||
signed = plan.get("signed")
|
||||
if signed:
|
||||
params = self.verify_signed(signed, trust)
|
||||
want = sorted(f"{e.get('name')}={e.get('version')}" for e in params.get("packages") or [])
|
||||
got = sorted(f"{e['name']}={e['version']}" for e in plan.get("packages", []))
|
||||
if not want or want != got:
|
||||
raise Refused("R3", "the plan's packages are not exactly the signed job's packages")
|
||||
if bool(params.get("undo")) != bool(plan.get("undo")):
|
||||
raise Refused("R3", "the plan's undo flag is not the signed job's")
|
||||
if params.get("vmid") not in (None, self.vmid):
|
||||
raise Refused("R3", "the signed job names another guest")
|
||||
return "signed", bool(plan.get("undo"))
|
||||
if plan.get("undo"):
|
||||
raise Refused("R3", "an undo (downgrade) needs a signed operator job")
|
||||
if trust.get("ring0_slow_lane") is True:
|
||||
return "ring0", False
|
||||
raise Refused("R3", "a Docker step needs a signed operator job (ring 1) or this box's root-owned ring-0 mark")
|
||||
|
||||
def live_restore(self):
|
||||
rc, out, _ = self.g(["docker", "info", "--format", "{{.LiveRestoreEnabled}}"], timeout=60)
|
||||
return out.strip() if rc == 0 and out.strip() in ("true", "false") else "unknown"
|
||||
|
||||
def container_ids(self):
|
||||
rc, out, _ = self.g(["docker", "ps", "-q", "--no-trunc"], timeout=60)
|
||||
return sorted(out.split()) if rc == 0 else None
|
||||
|
||||
def live_restore_on(self):
|
||||
"""`09` decision 87: merge live-restore into daemon.json and RELOAD (C5: a reload turns it on, no restart)."""
|
||||
log = self.r.log
|
||||
if self.live_restore() == "true":
|
||||
self.report["live_restore"] = {"result": "already on"}
|
||||
log("os-apply: LIVE-RESTORE already on")
|
||||
return 0
|
||||
rc, cur, _ = self.g(["cat", DAEMON_JSON], timeout=30)
|
||||
try:
|
||||
conf = json.loads(cur) if rc == 0 and cur.strip() else {}
|
||||
except ValueError:
|
||||
raise Refused("R16", f"{DAEMON_JSON} in the guest is not valid JSON — not touched")
|
||||
if not isinstance(conf, dict):
|
||||
raise Refused("R16", f"{DAEMON_JSON} is not a JSON object — not touched")
|
||||
before = self.container_ids()
|
||||
conf["live-restore"] = True
|
||||
self.r.write_file("guest", self.vmid, DAEMON_JSON, json.dumps(conf, indent=2, sort_keys=True) + "\n")
|
||||
rrc, _, rerr = self.g(["systemctl", "reload", "docker"], timeout=120)
|
||||
state = "unknown"
|
||||
for _ in range(10):
|
||||
state = self.live_restore()
|
||||
if state == "true":
|
||||
break
|
||||
self.r.sleep(1)
|
||||
after = self.container_ids()
|
||||
same = before is not None and before == after
|
||||
self.report["live_restore"] = {"result": "on" if state == "true" else "failed", "reload_rc": rrc,
|
||||
"containers_before": len(before or []), "same_ids": same}
|
||||
log(f"os-apply: LIVE-RESTORE reload_rc={rrc} state={state} containers={len(before or [])} same-ids={'yes' if same else 'NO'}")
|
||||
if state != "true":
|
||||
# put the old file back (and reload again) — still never a restart
|
||||
self.r.write_file("guest", self.vmid, DAEMON_JSON, cur if rc == 0 else "{}\n")
|
||||
self.g(["systemctl", "reload", "docker"], timeout=120)
|
||||
self.report["failed"] = {"rc": 3, "step": "live-restore", "reason": (rerr or "").strip()[-200:]}
|
||||
return 3
|
||||
return 0
|
||||
|
||||
def kernel_next_boot(self):
|
||||
"""Which kernel GRUB boots next, read without root-only files (grubenv + /etc/default/grub + /boot)."""
|
||||
try:
|
||||
dflt = re.search(r'^GRUB_DEFAULT=["\']?([^"\'\n]*)', self.r.read_file("/etc/default/grub"), re.M)
|
||||
dflt = dflt.group(1) if dflt else "0"
|
||||
except OSError:
|
||||
dflt = "0"
|
||||
env = {}
|
||||
try:
|
||||
for l in self.r.read_file("/boot/grub/grubenv").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, v = l.split("=", 1)
|
||||
env[k] = v
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def ver(entry):
|
||||
m = re.search(r"gnulinux-([0-9][^>\s]*?-pve)-(?:advanced|recovery)", entry)
|
||||
return m.group(1) if m else "unknown"
|
||||
if env.get("next_entry"):
|
||||
return ver(env["next_entry"]), "next_entry (a one-shot GRUB cannot clear on LVM /boot)"
|
||||
if dflt == "saved":
|
||||
return (ver(env["saved_entry"]), "saved default") if env.get("saved_entry") else ("unknown", "saved default unset")
|
||||
if dflt == "0":
|
||||
rc, out, _ = self.r.host(["sh", "-c", "ls /boot/vmlinuz-* 2>/dev/null"], 30)
|
||||
vers = [l.split("vmlinuz-", 1)[1] for l in out.split() if "vmlinuz-" in l]
|
||||
best = None
|
||||
for v in vers:
|
||||
if best is None or self.dpkg_cmp(v, "gt", best):
|
||||
best = v
|
||||
return (best or "unknown"), "GRUB_DEFAULT=0 (the newest installed)"
|
||||
return "unknown", f"GRUB_DEFAULT={dflt}"
|
||||
|
||||
def facts(self):
|
||||
"""Read-only versions for the hub's System page (R-852). A value that cannot be read is "unknown"."""
|
||||
def first(cmd, timeout=30):
|
||||
rc, out, _ = self.r.host(cmd, timeout)
|
||||
v = out.strip().splitlines()[0].strip() if rc == 0 and out.strip() else ""
|
||||
return v or "unknown"
|
||||
h = {"debian": first(["cat", "/etc/debian_version"]), "kernel_running": first(["uname", "-r"])}
|
||||
h["kernel_next_boot"], h["kernel_next_boot_source"] = self.kernel_next_boot()
|
||||
rc, out, _ = self.r.host(["apt-mark", "showhold"], 60)
|
||||
h["held"] = sorted(out.split()) if rc == 0 else None
|
||||
try:
|
||||
t = int(self.r.read_file("/proc/sys/kernel/tainted").strip())
|
||||
h["tainted"], h["oops_this_boot"], h["warn_this_boot"] = t, bool(t & 128), bool(t & 512)
|
||||
except (OSError, ValueError):
|
||||
h["tainted"], h["oops_this_boot"], h["warn_this_boot"] = None, None, None
|
||||
try:
|
||||
h["kernel_panic"] = int(self.r.read_file("/proc/sys/kernel/panic").strip())
|
||||
except (OSError, ValueError):
|
||||
h["kernel_panic"] = None
|
||||
try:
|
||||
h["crash_guard"] = json.loads(self.r.read_file(CRASH_GUARD_STATE))
|
||||
except (OSError, ValueError):
|
||||
h["crash_guard"] = None
|
||||
g = {"debian": "unknown", "docker_engine": "unknown", "containerd": "unknown", "live_restore": "unknown"}
|
||||
try:
|
||||
self.check_guest(self.vmid)
|
||||
running = True
|
||||
except Refused as e:
|
||||
running, g["unknown_reason"] = False, f"{e.code} {e.reason}"
|
||||
if running:
|
||||
script = ('echo "debian=$(cat /etc/debian_version 2>/dev/null)"; '
|
||||
'echo "engine=$(docker version --format \'{{.Server.Version}}\' 2>/dev/null)"; '
|
||||
'echo "containerd=$(dpkg-query -W -f \'${Version}\' containerd.io 2>/dev/null)"; '
|
||||
'echo "live=$(docker info --format \'{{.LiveRestoreEnabled}}\' 2>/dev/null)"')
|
||||
rc, out, _ = self.g(["sh", "-c", script], timeout=60)
|
||||
kv = dict(l.split("=", 1) for l in out.splitlines() if "=" in l)
|
||||
for k, src in (("debian", "debian"), ("docker_engine", "engine"), ("containerd", "containerd")):
|
||||
g[k] = kv.get(src, "").strip() or "unknown"
|
||||
g["live_restore"] = {"true": "on", "false": "off"}.get(kv.get("live", "").strip(), "unknown")
|
||||
self.report["facts"] = {"host": h, "guest": g}
|
||||
return 0
|
||||
|
||||
def check_guest(self, vmid):
|
||||
if vmid in RESERVED_VMIDS:
|
||||
raise Refused("R10", f"vmid {vmid} is a reserved scratch vmid")
|
||||
@@ -227,7 +502,7 @@ class Apply:
|
||||
"""Run in the TARGET layer: the guest via pct exec, or the host directly."""
|
||||
if self.layer == "host":
|
||||
return self.r.host(argv, timeout)
|
||||
return self.r.guest(self.vmid, argv, timeout)
|
||||
return self.r.guest(self.vmid, argv, timeout) # guest and docker both live in the customer guest
|
||||
|
||||
def g(self, argv, timeout=1800):
|
||||
"""Run in the customer GUEST whatever the layer (its health)."""
|
||||
@@ -289,21 +564,23 @@ class Apply:
|
||||
|
||||
def guest_health(self):
|
||||
"""The guest's signals: every container's state + health, the controller's own health, the network."""
|
||||
rc, out, _ = self.g(["docker", "ps", "-a", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}"], timeout=60)
|
||||
rc, out, _ = self.g(["docker", "ps", "-a", "--no-trunc", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.ID}}"], timeout=60)
|
||||
cont = {}
|
||||
for l in out.splitlines():
|
||||
p = l.split("\t")
|
||||
if len(p) == 3:
|
||||
if len(p) >= 3:
|
||||
h = "healthy" if "(healthy)" in p[2] else "unhealthy" if "(unhealthy)" in p[2] else \
|
||||
"starting" if "(health: starting)" in p[2] else "none"
|
||||
cont[p[0]] = {"state": p[1], "health": h}
|
||||
if len(p) >= 4 and p[3]:
|
||||
cont[p[0]]["id"] = p[3]
|
||||
nrc, _, _ = self.g(["getent", "hosts", "deb.debian.org"], timeout=30)
|
||||
return {"docker_ok": rc == 0, "containers": cont,
|
||||
"controller": cont.get("felhom-controller", {}).get("health", "absent"),
|
||||
"network_ok": nrc == 0}
|
||||
|
||||
def health(self):
|
||||
if self.layer == "guest":
|
||||
if self.layer in ("guest", "docker"):
|
||||
return self.guest_health()
|
||||
rc, out, _ = self.r.host(["systemctl", "is-active"] + HOST_SERVICES, 30)
|
||||
states = out.split()
|
||||
@@ -315,7 +592,7 @@ class Apply:
|
||||
def restart_needed(self):
|
||||
"""Processes still mapping deleted files, OUTSIDE containers (C11). Guest: outside docker; host: outside the
|
||||
LXC guests (the host's /proc shows guest processes too)."""
|
||||
skip = RESTART_SKIP_CGROUP[self.layer]
|
||||
skip = RESTART_SKIP_CGROUP["host" if self.layer == "host" else "guest"]
|
||||
script = ('for p in /proc/[0-9]*; do grep -q "(deleted)" $p/maps 2>/dev/null || continue; '
|
||||
'grep -q "%s" $p/cgroup 2>/dev/null && continue; echo "${p#/proc/} $(cat $p/comm 2>/dev/null)"; done' % skip)
|
||||
rc, out, _ = self.x(["sh", "-c", script], timeout=120)
|
||||
@@ -373,16 +650,28 @@ class Apply:
|
||||
plan = self.load_plan()
|
||||
self.mode, self.layer, self.vmid, self.select = self.check_plan(plan)
|
||||
self.report.update(mode=self.mode, layer=self.layer, release_id=plan.get("release_id"), vmid=self.vmid)
|
||||
if self.mode == "facts":
|
||||
return self.facts()
|
||||
if self.layer == "host":
|
||||
self.check_appliance()
|
||||
self.check_guest(self.vmid)
|
||||
log = self.r.log
|
||||
if self.mode == "live-restore-on":
|
||||
return self.live_restore_on()
|
||||
self.who, self.allow_downgrade = ("fast", False)
|
||||
if self.layer == "docker" and self.mode == "apply":
|
||||
self.who, self.allow_downgrade = self.docker_authority(plan)
|
||||
if self.live_restore() != "true":
|
||||
raise Refused("R15", "live-restore is not ON in the guest — a Docker step would restart every container")
|
||||
self.report["authority"] = self.who
|
||||
self.report["undo"] = self.allow_downgrade
|
||||
if self.mode == "health":
|
||||
self.report["health"] = self.health()
|
||||
return 0
|
||||
log(f"os-apply: START release={plan.get('release_id')} layer={self.layer}" +
|
||||
(f":{self.vmid}" if self.layer == "guest" else "") +
|
||||
f" lane=fast mode={self.mode} select={self.select} packages={len(plan.get('packages', []))}")
|
||||
(f":{self.vmid}" if self.layer != "host" else "") +
|
||||
f" lane={plan.get('lane', 'fast')} mode={self.mode} select={self.select} packages={len(plan.get('packages', []))}" +
|
||||
(f" authority={self.who}{' UNDO' if self.allow_downgrade else ''}" if self.layer == "docker" else ""))
|
||||
if self.apt_lock_held():
|
||||
raise Refused("R9", f"another apt/dpkg holds the lock on the {self.layer}")
|
||||
self.report["health_before"] = self.health()
|
||||
@@ -397,11 +686,15 @@ class Apply:
|
||||
if rc:
|
||||
return rc
|
||||
self.report.update(self.inventory(installed_after))
|
||||
if self.layer == "host" and "reboot_needed" not in self.report:
|
||||
# The host is scanned on EVERY pass (local, no pct exec): a reboot must CLEAR "reboot needed", or the hub's
|
||||
# 14-day alarm fires on a host that was rebooted long ago. The guest still scans only after an install
|
||||
# (R-845; one pct exec, and no alarm reads it). Pinned by test_host_scans_every_pass_guest_only_after_install.
|
||||
if "reboot_needed" not in self.report:
|
||||
# EVERY layer is scanned on EVERY pass: a reboot (host) or a restart (guest) must CLEAR "restart needed",
|
||||
# or the fleet view keeps a stale date (R-849, v0.142.0; the host since v0.141.1). One pct exec, ~1 s.
|
||||
# Pinned by test_every_layer_scans_every_pass.
|
||||
self.report["restart_needed"], self.report["reboot_needed"] = self.restart_needed()
|
||||
self.report["docker_restart_needed"] = any(p in ("dockerd", "containerd") for p in self.report["restart_needed"])
|
||||
if self.layer == "docker":
|
||||
rc_v, out_v, _ = self.g(["docker", "version", "--format", "{{.Server.Version}}"], timeout=60)
|
||||
self.report["docker_engine"] = out_v.strip() if rc_v == 0 and out_v.strip() else "unknown"
|
||||
self.report["reboot_scanned"] = "reboot_needed" in self.report
|
||||
self.report["health_after"] = self.health()
|
||||
return 0
|
||||
@@ -435,9 +728,27 @@ class Apply:
|
||||
out.append({"name": p["name"], "version": p["to"], "origin": "Debian-Security" if "Debian-Security" in o else "Debian"})
|
||||
return out
|
||||
|
||||
def pending_docker(self):
|
||||
"""Ring 0 (select pending-docker): the newest pending version of each INSTALLED Docker package, Docker origin."""
|
||||
rc, pend, remv, _ = self.simulate(["dist-upgrade"])
|
||||
return [{"name": p["name"], "version": p["to"], "origin": DOCKER_ORIGIN} for p in pend
|
||||
if p["from"] is not None and p["name"] in DOCKER_NAMES and self.origin_name(p["origin"]) == {DOCKER_ORIGIN}]
|
||||
|
||||
def origin_ok(self, origin):
|
||||
o = self.origin_name(origin)
|
||||
if self.layer == "docker":
|
||||
return o == {DOCKER_ORIGIN}
|
||||
return bool(o & set(FAST_ORIGINS))
|
||||
|
||||
def apply(self, plan):
|
||||
log = self.r.log
|
||||
packages = plan["packages"] if self.select == "listed" else self.pending_fast()
|
||||
if self.select == "listed":
|
||||
packages = plan["packages"]
|
||||
elif self.select == "pending-docker":
|
||||
packages = self.pending_docker()
|
||||
else:
|
||||
packages = self.pending_fast()
|
||||
cmp_op = "ne" if self.allow_downgrade else "gt"
|
||||
inst = self.installed()
|
||||
upgrade, already, notinst = [], 0, 0
|
||||
for e in packages:
|
||||
@@ -445,7 +756,7 @@ class Apply:
|
||||
if n not in inst:
|
||||
notinst += 1
|
||||
continue
|
||||
if not self.dpkg_cmp(v, "gt", inst[n]):
|
||||
if not self.dpkg_cmp(v, cmp_op, inst[n]):
|
||||
already += 1
|
||||
continue
|
||||
upgrade.append((n, v))
|
||||
@@ -473,7 +784,8 @@ class Apply:
|
||||
self.report["upgraded"] = []
|
||||
log("os-apply: DONE rc=0 seconds=0 upgraded=0 (nothing to do)")
|
||||
return 0, inst
|
||||
args = ["install", "--only-upgrade", "--no-install-recommends"] + [f"{n}={v}" for n, v in upgrade]
|
||||
args = ["install", "--only-upgrade", "--no-install-recommends"] + \
|
||||
(["--allow-downgrades"] if self.allow_downgrade else []) + [f"{n}={v}" for n, v in upgrade]
|
||||
rc, sim, remv, text = self.simulate(args)
|
||||
if rc != 0:
|
||||
tail = text.strip().splitlines()[-1] if text.strip() else ""
|
||||
@@ -488,10 +800,10 @@ class Apply:
|
||||
raise Refused("R6", f"the plan would touch {p['name']}, which is not in the plan")
|
||||
if p["to"] != want[p["name"]]:
|
||||
raise Refused("R6", f"{p['name']} would go to {p['to']}, not the approved {want[p['name']]}")
|
||||
if not self.dpkg_cmp(p["to"], "gt", p["from"]):
|
||||
if not self.allow_downgrade and not self.dpkg_cmp(p["to"], "gt", p["from"]):
|
||||
raise Refused("R5", f"{p['name']} would be downgraded {p['from']} -> {p['to']}")
|
||||
if not self.origin_name(p["origin"]) & set(FAST_ORIGINS):
|
||||
raise Refused("R2", f"{p['name']} would come from {p['origin']}, not Debian")
|
||||
if not self.origin_ok(p["origin"]):
|
||||
raise Refused("R2", f"{p['name']} would come from {p['origin']}, not the {self.layer} layer's origin")
|
||||
if self.layer == "host" and HOST_SLOW_RE.match(p["name"]):
|
||||
raise Refused("R14", f"{p['name']} is a kernel / boot / firmware package — the host's slow lane")
|
||||
need = self.download_bytes(args)
|
||||
@@ -525,7 +837,7 @@ class Apply:
|
||||
return 3, None
|
||||
self.report["upgraded"] = [{"name": n, "version": v} for n, v in upgrade]
|
||||
self.report["seconds"] = round(secs, 1)
|
||||
procs, reboot = self.restart_needed() # only after an install (R-845)
|
||||
procs, reboot = self.restart_needed()
|
||||
self.report["restart_needed"] = procs
|
||||
self.report["docker_restart_needed"] = any(p in ("dockerd", "containerd") for p in procs)
|
||||
self.report["reboot_needed"] = reboot
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/python3
|
||||
"""Tests for felhom-crash-guard (`11` §5.9). Temp dirs only; nothing real is touched. Red-proof seam: CRASHGUARD_UNDER_TEST."""
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
_loader = importlib.machinery.SourceFileLoader("crashguard", os.environ.get("CRASHGUARD_UNDER_TEST", str(HERE / "felhom-crash-guard")))
|
||||
_spec = importlib.util.spec_from_loader("crashguard", _loader)
|
||||
cg = importlib.util.module_from_spec(_spec)
|
||||
_loader.exec_module(cg)
|
||||
|
||||
T0 = 1791115200.0 # 2026-10-04T12:00:00Z
|
||||
|
||||
|
||||
class FakeEnv(cg.Env):
|
||||
def __init__(self, d):
|
||||
super().__init__(conf=os.path.join(d, "conf"), state_dir=os.path.join(d, "state"),
|
||||
panic_path=os.path.join(d, "panic"), uptime_path=os.path.join(d, "uptime"),
|
||||
boot_id_path=os.path.join(d, "bootid"))
|
||||
self.t = T0
|
||||
self.logs = []
|
||||
open(self.panic_path, "w").write("0\n")
|
||||
open(self.uptime_path, "w").write("20.00 10.00\n")
|
||||
|
||||
def now(self):
|
||||
return self.t
|
||||
|
||||
def log(self, line):
|
||||
self.logs.append(line)
|
||||
|
||||
def panic(self):
|
||||
return int(open(self.panic_path).read())
|
||||
|
||||
def state(self):
|
||||
return json.load(open(os.path.join(self.state_dir, "state.json")))
|
||||
|
||||
|
||||
class Guard(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.d = tempfile.TemporaryDirectory()
|
||||
self.e = FakeEnv(self.d.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.d.cleanup()
|
||||
|
||||
def crash_boot(self, minutes_later):
|
||||
self.e.t += minutes_later * 60
|
||||
cg.main(["x", "boot"], self.e) # no clean-stop before it: an unclean stop
|
||||
|
||||
def clean_reboot(self, minutes_later):
|
||||
cg.main(["x", "clean-stop"], self.e)
|
||||
self.e.t += minutes_later * 60
|
||||
cg.main(["x", "boot"], self.e)
|
||||
|
||||
def test_first_boot_is_not_a_crash_and_arms(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["last_boot_unclean"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
self.assertTrue(s["armed"])
|
||||
|
||||
def test_clean_reboots_never_count(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
for _ in range(5):
|
||||
self.clean_reboot(1)
|
||||
s = self.e.state()
|
||||
self.assertEqual(s["unclean_boots_in_window"], 0)
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
|
||||
def test_third_crash_in_an_hour_leaves_the_box_off(self):
|
||||
# operator's words: "if it crashes 3 times within one hour, it stays off" — after crash 2 the guard trips,
|
||||
# so crash 3 (kernel.panic = 0) does not restart the box.
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.assertEqual(self.e.panic(), 10, "one crash: still restarts")
|
||||
self.crash_boot(5)
|
||||
s = self.e.state()
|
||||
self.assertTrue(s["tripped"], s)
|
||||
self.assertEqual(self.e.panic(), 0, "after the 2nd crash boot the 3rd crash must leave the box off")
|
||||
self.assertIn("2 unclean boots within 60 minutes", s["tripped_reason"])
|
||||
|
||||
def test_crashes_spread_over_more_than_the_window_do_not_trip(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(61)
|
||||
self.assertFalse(self.e.state()["tripped"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
|
||||
def test_tripped_stays_tripped_across_boots(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.clean_reboot(30) # the operator switched it on; even a clean boot keeps the trip
|
||||
self.assertTrue(self.e.state()["tripped"])
|
||||
self.assertEqual(self.e.panic(), 0)
|
||||
|
||||
def test_rearms_after_24h_of_normal_running(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.e.t += 23 * 3600
|
||||
cg.main(["x", "check"], self.e)
|
||||
self.assertTrue(self.e.state()["tripped"], "not before 24 h")
|
||||
self.e.t += 3600
|
||||
cg.main(["x", "check"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["tripped"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
self.assertIn("timer", s["rearmed_by"])
|
||||
|
||||
def test_operator_rearm_starts_a_fresh_window(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.e.t += 60
|
||||
cg.main(["x", "rearm"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["tripped"])
|
||||
self.assertEqual(s["rearmed_by"], "operator")
|
||||
self.assertEqual(s["unclean_boots_24h"], 2, "the history stays")
|
||||
self.crash_boot(5)
|
||||
self.assertFalse(self.e.state()["tripped"], "one crash after a re-arm must not trip at once")
|
||||
|
||||
def test_config_numbers_are_read(self):
|
||||
open(self.e.conf, "w").write("LIMIT=2\nPANIC_SECONDS=30\n")
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.assertEqual(self.e.panic(), 30)
|
||||
self.crash_boot(1)
|
||||
self.assertTrue(self.e.state()["tripped"], "LIMIT=2: the first crash boot trips")
|
||||
|
||||
def test_state_is_world_readable_for_the_agent(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
mode = os.stat(os.path.join(self.e.state_dir, "state.json")).st_mode & 0o777
|
||||
self.assertEqual(mode, 0o644)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+331
-23
@@ -64,6 +64,35 @@ class Fake:
|
||||
self.services = {}
|
||||
self.files[osapply.INSTALL_STATE] = json.dumps({"mode": "appliance"})
|
||||
self.stats[osapply.INSTALL_STATE] = St(mode=statmod.S_IFREG | 0o644, uid=0)
|
||||
# Docker / trust / facts state (v0.142.0)
|
||||
self.live_restore = "true"
|
||||
self.ids = ["aaa111", "bbb222"]
|
||||
self.engine = "29.7.2"
|
||||
self.daemon_json = '{"log-driver": "json-file"}'
|
||||
self.reload_enables = True
|
||||
self.sig_rc = 0
|
||||
self.nonces = {}
|
||||
self.clock = 1791115200.0 # 2026-10-04T12:00:00Z
|
||||
self.files[osapply.TRUST_FILE] = json.dumps({"host_id": "demo-hp-bb76ea", "ring0_slow_lane": False})
|
||||
self.stats[osapply.TRUST_FILE] = St(mode=statmod.S_IFREG | 0o644, uid=0)
|
||||
self.files[osapply.TRUST_SIGNERS] = 'felhom-op-1 namespaces="felhom-op-v1" ssh-ed25519 AAAA\n'
|
||||
self.stats[osapply.TRUST_SIGNERS] = St(mode=statmod.S_IFREG | 0o644, uid=0)
|
||||
|
||||
def now(self):
|
||||
return self.clock
|
||||
|
||||
def sleep(self, s):
|
||||
pass
|
||||
|
||||
def verify_sig(self, signers, key_id, ns, blob, sig):
|
||||
self.verified = (signers, key_id, ns, blob, sig)
|
||||
return self.sig_rc
|
||||
|
||||
def read_nonces(self):
|
||||
return dict(self.nonces)
|
||||
|
||||
def write_nonces(self, d):
|
||||
self.nonces = dict(d)
|
||||
|
||||
# Runner interface
|
||||
def read_file(self, p):
|
||||
@@ -151,11 +180,38 @@ class Fake:
|
||||
return 0, getattr(self, "install_out", "Setting up libc6 ...\n"), ""
|
||||
if cmd == "df":
|
||||
return 0, f"Avail\n{self.free}\n", ""
|
||||
if cmd == "docker" and a[1] == "info":
|
||||
return 0, self.live_restore + "\n", ""
|
||||
if cmd == "docker" and a[1] == "version":
|
||||
return 0, self.engine + "\n", ""
|
||||
if cmd == "docker" and a[1:3] == ["ps", "-q"]:
|
||||
return 0, "".join(i + "\n" for i in self.ids), ""
|
||||
if cmd == "docker":
|
||||
return 0, "felhom-controller\trunning\tUp 1 hour (healthy)\napp\trunning\tUp 1 hour (healthy)\n", ""
|
||||
ids = self.ids + ["x"] * 2
|
||||
return 0, f"felhom-controller\trunning\tUp 1 hour (healthy)\t{ids[0]}\napp\trunning\tUp 1 hour (healthy)\t{ids[1]}\n", ""
|
||||
if cmd == "cat" and a[1] == osapply.DAEMON_JSON:
|
||||
return (0, self.daemon_json, "") if self.daemon_json is not None else (1, "", "No such file")
|
||||
if cmd == "cat" and a[1] == "/etc/debian_version":
|
||||
return 0, "13.7\n", ""
|
||||
if cmd == "uname":
|
||||
return 0, "7.0.14-20-pve\n", ""
|
||||
if cmd == "apt-mark":
|
||||
return 0, getattr(self, "held", ""), ""
|
||||
if cmd == "systemctl" and a[1] == "reload":
|
||||
self.reloads = getattr(self, "reloads", 0) + 1
|
||||
if self.reload_enables and '"live-restore": true' in self.written.get(osapply.DAEMON_JSON, ""):
|
||||
self.live_restore = "true"
|
||||
return 0, "", ""
|
||||
if cmd == "systemctl" and a[1] == "restart":
|
||||
self.restarted = True
|
||||
return 0, "", ""
|
||||
if cmd == "getent":
|
||||
return 0, "1.2.3.4 deb.debian.org\n", ""
|
||||
if cmd == "sh":
|
||||
if "vmlinuz" in a[2]:
|
||||
return 0, "/boot/vmlinuz-7.0.2-6-pve\n/boot/vmlinuz-7.0.14-20-pve\n", ""
|
||||
if "engine=" in a[2]:
|
||||
return 0, f"debian=13.7\nengine={self.engine}\ncontainerd=2.3.3-1~debian.13~trixie\nlive={self.live_restore}\n", ""
|
||||
if "os-release" in a[2]:
|
||||
return 0, "trixie\n", ""
|
||||
if "(deleted)" in a[2]:
|
||||
@@ -180,7 +236,7 @@ class Fake:
|
||||
n, v = x.split("=", 1)
|
||||
if v not in self.avail(n):
|
||||
return 100, "", f"E: Version '{v}' for '{n}' was not found"
|
||||
origin = SEC if n == "openssl" else DEB
|
||||
origin = "Docker CE:trixie" if n in osapply.DOCKER_NAMES else SEC if n == "openssl" else DEB
|
||||
out += f"Inst {n} [{self.installed[n]}] ({v} {origin} [amd64])\n"
|
||||
out += "".join(l + "\n" for l in self.extra_sim)
|
||||
return 0, out, ""
|
||||
@@ -226,7 +282,6 @@ class Happy(unittest.TestCase):
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(f.installed["libc6"], "2.41-12+deb13u3")
|
||||
self.assertIn("installed", rep)
|
||||
self.assertNotIn("restart_needed", rep, "the restart scan runs only after an install (R-845)")
|
||||
|
||||
def test_health_mode(self):
|
||||
f = Fake()
|
||||
@@ -551,26 +606,22 @@ class HostLayer(unittest.TestCase):
|
||||
rc, rep = run(f)
|
||||
self.assertTrue(rep["reboot_needed"], rep)
|
||||
|
||||
def test_host_scans_every_pass_guest_only_after_install(self):
|
||||
# A host pass that installs nothing still scans, so a reboot clears the flag (the hub alarm reads it).
|
||||
f = Fake()
|
||||
f.plan["layer"] = "host"
|
||||
f.plan["mode"] = "inventory"
|
||||
f.restart_out = "2101 lxc-start\n"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(rep["reboot_scanned"], rep)
|
||||
self.assertTrue(rep["reboot_needed"], rep)
|
||||
f = Fake()
|
||||
f.plan["layer"] = "host"
|
||||
f.plan["mode"] = "inventory"
|
||||
f.restart_out = "" # after the reboot: nothing maps a deleted file
|
||||
rc, rep = run(f)
|
||||
self.assertTrue(rep["reboot_scanned"] and rep["reboot_needed"] is False, rep)
|
||||
f = Fake()
|
||||
f.plan["mode"] = "inventory"
|
||||
rc, rep = run(f)
|
||||
self.assertFalse(rep["reboot_scanned"], "the guest scans only after an install (R-845)")
|
||||
def test_every_layer_scans_every_pass(self):
|
||||
# R-849 (v0.142.0): host AND guest are scanned on every pass, so a reboot / restart clears the flag.
|
||||
for layer in ("host", "guest"):
|
||||
f = Fake()
|
||||
f.plan["layer"] = layer
|
||||
f.plan["mode"] = "inventory"
|
||||
f.restart_out = "2101 lxc-start\n" if layer == "host" else "1 systemd\n"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(rep["reboot_scanned"] and rep["reboot_needed"], (layer, rep))
|
||||
f = Fake()
|
||||
f.plan["layer"] = layer
|
||||
f.plan["mode"] = "inventory"
|
||||
f.restart_out = ""
|
||||
rc, rep = run(f)
|
||||
self.assertTrue(rep["reboot_scanned"] and rep["reboot_needed"] is False, (layer, rep))
|
||||
|
||||
def test_no_reboot_for_ordinary_daemons(self):
|
||||
f = Fake()
|
||||
@@ -624,5 +675,262 @@ class RestartSkipPattern(unittest.TestCase):
|
||||
self.assertTrue(self.grep(guest, "0::/system.slice/docker-0123abcd.scope"))
|
||||
self.assertFalse(self.grep(guest, "0::/system.slice/cron.service"))
|
||||
|
||||
|
||||
DOCKER_SET = [{"name": "docker-ce", "version": "5:29.8.2-1~debian.13~trixie", "origin": "Docker CE"},
|
||||
{"name": "containerd.io", "version": "2.3.6-1~debian.13~trixie", "origin": "Docker CE"}]
|
||||
|
||||
|
||||
def docker_fake(signed=None, undo=False, ring0=False):
|
||||
f = Fake()
|
||||
f.installed.update({"docker-ce": "5:29.7.2-1~debian.13~trixie", "containerd.io": "2.3.3-1~debian.13~trixie"})
|
||||
f.live["docker-ce"] = {"5:29.8.2-1~debian.13~trixie", "5:29.7.2-1~debian.13~trixie"}
|
||||
f.live["containerd.io"] = {"2.3.6-1~debian.13~trixie", "2.3.3-1~debian.13~trixie"}
|
||||
f.plan = {"release_id": "os-docker-t1", "layer": "docker", "lane": "slow", "vmid": 9201, "mode": "apply",
|
||||
"packages": [dict(p) for p in DOCKER_SET]}
|
||||
if undo:
|
||||
f.plan["undo"] = True
|
||||
if ring0:
|
||||
f.files[osapply.TRUST_FILE] = json.dumps({"host_id": "demo-hp-bb76ea", "ring0_slow_lane": True})
|
||||
if signed is not None:
|
||||
f.plan["signed"] = signed
|
||||
return f
|
||||
|
||||
|
||||
def signed_job(packages=DOCKER_SET, host="demo-hp-bb76ea", op="os_docker_step", undo=False, nonce="n1",
|
||||
issued="2026-10-04T11:50:00Z", expires="2026-10-04T12:30:00Z"):
|
||||
import base64
|
||||
params = {"packages": packages, "undo": undo}
|
||||
blob = json.dumps({"expires_at": expires, "issued_at": issued, "key_id": "felhom-op-1", "nonce": nonce, "op": op,
|
||||
"params": params, "target": {"guest_id": "", "host_id": host}}, sort_keys=True).encode()
|
||||
return {"blob_b64": base64.b64encode(blob).decode(), "sig": "-----BEGIN SSH SIGNATURE-----\nx\n-----END SSH SIGNATURE-----\n"}
|
||||
|
||||
|
||||
class DockerLane(unittest.TestCase):
|
||||
"""`11` §5.8, agent v0.142.0. Each test names the refusal it pins; the red-proof file mutates each one."""
|
||||
|
||||
def refused(self, f, code):
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 2, rep)
|
||||
self.assertEqual(rep["refused"]["code"], code, rep)
|
||||
self.assertEqual(f.installed.get("docker-ce", "5:29.7.2-1~debian.13~trixie"), "5:29.7.2-1~debian.13~trixie")
|
||||
return rep
|
||||
|
||||
def test_docker_package_in_a_fast_plan_is_refused(self):
|
||||
f = Fake()
|
||||
f.plan["packages"].append({"name": "docker-ce", "version": "5:29.8.2-1~debian.13~trixie", "origin": "Debian"})
|
||||
self.refused(f, "R2")
|
||||
|
||||
def test_docker_layer_in_the_fast_lane_is_refused(self):
|
||||
f = docker_fake(ring0=True)
|
||||
f.plan["lane"] = "fast"
|
||||
self.refused(f, "R3")
|
||||
|
||||
def test_no_authority_is_refused(self):
|
||||
self.refused(docker_fake(), "R3")
|
||||
|
||||
def test_ring0_mark_allows_pending_docker(self):
|
||||
f = docker_fake(ring0=True)
|
||||
f.plan["select"], f.plan["packages"] = "pending-docker", []
|
||||
f.pending_sim = ["Inst docker-ce [5:29.7.2-1~debian.13~trixie] (5:29.8.2-1~debian.13~trixie Docker CE:trixie [amd64])",
|
||||
"Inst containerd.io [2.3.3-1~debian.13~trixie] (2.3.6-1~debian.13~trixie Docker CE:trixie [amd64])",
|
||||
"Inst bash [5.2.37-2+b9] (5.2.37-2+b10 Debian:13.7/stable [amd64])"]
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(rep["authority"], "ring0")
|
||||
self.assertEqual(f.installed["docker-ce"], "5:29.8.2-1~debian.13~trixie")
|
||||
self.assertEqual(f.installed["bash"], "5.2.37-2+b9", "a Debian package must not ride a Docker step")
|
||||
self.assertFalse(getattr(f, "restarted", False), "never a docker restart")
|
||||
|
||||
def test_signed_job_applies_exactly_its_packages(self):
|
||||
f = docker_fake(signed=signed_job())
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(rep["authority"], "signed")
|
||||
self.assertEqual(f.installed["containerd.io"], "2.3.6-1~debian.13~trixie")
|
||||
self.assertEqual(f.verified[0], osapply.TRUST_SIGNERS, "the ROOT-OWNED signers file, not the agent's config")
|
||||
self.assertIn("n1", f.nonces)
|
||||
|
||||
def test_bad_signature_is_refused(self):
|
||||
f = docker_fake(signed=signed_job())
|
||||
f.sig_rc = 255
|
||||
self.refused(f, "R3")
|
||||
self.assertEqual(f.nonces, {}, "a bad signature must not burn a nonce")
|
||||
|
||||
def test_signed_job_for_another_host_is_refused(self):
|
||||
self.refused(docker_fake(signed=signed_job(host="demo-felhom-8363b5")), "R3")
|
||||
|
||||
def test_signed_job_other_op_is_refused(self):
|
||||
self.refused(docker_fake(signed=signed_job(op="agent_update")), "R3")
|
||||
|
||||
def test_expired_signed_job_is_refused(self):
|
||||
self.refused(docker_fake(signed=signed_job(expires="2026-10-04T11:55:00Z")), "R3")
|
||||
|
||||
def test_replayed_signed_job_is_refused(self):
|
||||
f = docker_fake(signed=signed_job())
|
||||
f.nonces = {"n1": f.clock + 600}
|
||||
self.refused(f, "R3")
|
||||
|
||||
def test_plan_must_equal_the_signed_packages(self):
|
||||
f = docker_fake(signed=signed_job(packages=DOCKER_SET[:1]))
|
||||
self.refused(f, "R3")
|
||||
|
||||
def test_agent_writable_trust_file_is_refused(self):
|
||||
f = docker_fake(ring0=True)
|
||||
f.stats[osapply.TRUST_FILE] = St(mode=statmod.S_IFREG | 0o644, uid=999)
|
||||
self.refused(f, "R3")
|
||||
|
||||
def test_live_restore_off_is_refused(self):
|
||||
f = docker_fake(signed=signed_job())
|
||||
f.live_restore = "false"
|
||||
self.refused(f, "R15")
|
||||
|
||||
def test_undo_needs_a_signed_job(self):
|
||||
self.refused(docker_fake(undo=True, ring0=True), "R3")
|
||||
|
||||
def test_signed_undo_downgrades(self):
|
||||
old = [{"name": "docker-ce", "version": "5:29.7.2-1~debian.13~trixie", "origin": "Docker CE"}]
|
||||
f = docker_fake(signed=signed_job(packages=old, undo=True), undo=True)
|
||||
f.plan["packages"] = [dict(p) for p in old]
|
||||
f.installed["docker-ce"] = "5:29.8.2-1~debian.13~trixie"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(f.installed["docker-ce"], "5:29.7.2-1~debian.13~trixie")
|
||||
self.assertTrue(rep["undo"])
|
||||
|
||||
def test_unsigned_downgrade_is_refused(self):
|
||||
f = docker_fake(ring0=True)
|
||||
f.plan["packages"] = [{"name": "docker-ce", "version": "5:29.6.0-1~debian.13~trixie", "origin": "Docker CE"}]
|
||||
f.live["docker-ce"].add("5:29.6.0-1~debian.13~trixie")
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rep["plan"]["upgrade"], 0, "an older version on an unsigned step is 'already', never installed")
|
||||
|
||||
def test_health_carries_container_ids(self):
|
||||
f = docker_fake(signed=signed_job())
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rep["health_after"]["containers"]["app"]["id"], "bbb222")
|
||||
self.assertEqual(rep["docker_engine"], "29.7.2")
|
||||
|
||||
|
||||
class LiveRestore(unittest.TestCase):
|
||||
def lr(self):
|
||||
f = Fake()
|
||||
f.plan = {"release_id": "lr", "layer": "guest", "lane": "fast", "vmid": 9201, "mode": "live-restore-on", "packages": []}
|
||||
f.live_restore = "false"
|
||||
return f
|
||||
|
||||
def test_turns_it_on_with_a_reload_never_a_restart(self):
|
||||
f = self.lr()
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(json.loads(f.written[osapply.DAEMON_JSON]), {"log-driver": "json-file", "live-restore": True})
|
||||
self.assertEqual(f.reloads, 1)
|
||||
self.assertFalse(getattr(f, "restarted", False))
|
||||
self.assertEqual(rep["live_restore"]["result"], "on")
|
||||
self.assertTrue(rep["live_restore"]["same_ids"])
|
||||
|
||||
def test_already_on_writes_nothing(self):
|
||||
f = self.lr()
|
||||
f.live_restore = "true"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertNotIn(osapply.DAEMON_JSON, f.written)
|
||||
|
||||
def test_invalid_daemon_json_is_left_alone(self):
|
||||
f = self.lr()
|
||||
f.daemon_json = "{not json"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rep["refused"]["code"], "R16")
|
||||
self.assertNotIn(osapply.DAEMON_JSON, f.written)
|
||||
|
||||
def test_reload_that_does_not_enable_puts_the_file_back(self):
|
||||
f = self.lr()
|
||||
f.reload_enables = False
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assertEqual(f.written[osapply.DAEMON_JSON], '{"log-driver": "json-file"}')
|
||||
self.assertFalse(getattr(f, "restarted", False))
|
||||
|
||||
|
||||
class Facts(unittest.TestCase):
|
||||
def facts(self, f=None):
|
||||
f = f or Fake()
|
||||
f.plan = {"release_id": "facts", "layer": "host", "lane": "fast", "vmid": 9201, "mode": "facts", "packages": []}
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
return f, rep["facts"]
|
||||
|
||||
def test_reads_host_and_guest(self):
|
||||
f = Fake()
|
||||
f.files["/proc/sys/kernel/tainted"] = "4225\n" # 4096 + 128 (D: oops) + 1
|
||||
f.files["/proc/sys/kernel/panic"] = "10\n"
|
||||
f.held = "tzdata\n"
|
||||
f.files["/etc/default/grub"] = "GRUB_DEFAULT=saved\n"
|
||||
f.files["/boot/grub/grubenv"] = "# GRUB Environment Block\nsaved_entry=gnulinux-advanced-x>gnulinux-7.0.14-20-pve-advanced-x\n"
|
||||
_, fa = self.facts(f)
|
||||
h, g = fa["host"], fa["guest"]
|
||||
self.assertEqual((h["debian"], h["kernel_running"], h["kernel_next_boot"]), ("13.7", "7.0.14-20-pve", "7.0.14-20-pve"))
|
||||
self.assertEqual(h["held"], ["tzdata"])
|
||||
self.assertTrue(h["oops_this_boot"])
|
||||
self.assertEqual(h["kernel_panic"], 10)
|
||||
self.assertEqual((g["debian"], g["docker_engine"], g["live_restore"]), ("13.7", "29.7.2", "on"))
|
||||
self.assertEqual(g["containerd"], "2.3.3-1~debian.13~trixie")
|
||||
|
||||
def test_next_entry_wins_and_default_zero_is_the_newest(self):
|
||||
f = Fake()
|
||||
f.files["/etc/default/grub"] = "GRUB_DEFAULT=saved\n"
|
||||
f.files["/boot/grub/grubenv"] = "saved_entry=gnulinux-advanced-x>gnulinux-7.0.2-6-pve-advanced-x\nnext_entry=gnulinux-advanced-x>gnulinux-7.0.14-20-pve-advanced-x\n"
|
||||
_, fa = self.facts(f)
|
||||
self.assertEqual(fa["host"]["kernel_next_boot"], "7.0.14-20-pve")
|
||||
self.assertIn("next_entry", fa["host"]["kernel_next_boot_source"])
|
||||
f = Fake()
|
||||
f.files["/etc/default/grub"] = "GRUB_DEFAULT=0\n"
|
||||
_, fa = self.facts(f)
|
||||
self.assertEqual(fa["host"]["kernel_next_boot"], "7.0.14-20-pve", "dpkg order, not string order (7.0.2 < 7.0.14)")
|
||||
|
||||
def test_stopped_guest_is_unknown_never_guessed(self):
|
||||
f = Fake()
|
||||
f.status = "status: stopped"
|
||||
_, fa = self.facts(f)
|
||||
self.assertEqual(fa["guest"]["docker_engine"], "unknown")
|
||||
self.assertIn("R10", fa["guest"]["unknown_reason"])
|
||||
self.assertEqual(fa["host"]["tainted"], None, "unreadable -> None, not 0")
|
||||
|
||||
|
||||
class RealSignatureCheck(unittest.TestCase):
|
||||
"""The REAL Runner.verify_sig with the real ssh-keygen and a throwaway key, in the installer's allowed_signers form
|
||||
(`<key_id> namespaces="felhom-op-v1" <type> <b64> <comment>`). Nothing leaves the temp dir."""
|
||||
|
||||
def setUp(self):
|
||||
import shutil
|
||||
if not shutil.which("ssh-keygen"):
|
||||
self.skipTest("ssh-keygen not available")
|
||||
import tempfile
|
||||
self.d = tempfile.mkdtemp()
|
||||
self.key = os.path.join(self.d, "k")
|
||||
subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "felhom-op-1", "-f", self.key], check=True)
|
||||
pub = open(self.key + ".pub").read().strip()
|
||||
self.signers = os.path.join(self.d, "signers")
|
||||
open(self.signers, "w").write(f'felhom-op-1 namespaces="felhom-op-v1" {pub}\n')
|
||||
|
||||
def sign(self, blob, ns="felhom-op-v1"):
|
||||
bp = os.path.join(self.d, "blob")
|
||||
open(bp, "wb").write(blob)
|
||||
if os.path.exists(bp + ".sig"):
|
||||
os.remove(bp + ".sig") # ssh-keygen -Y sign asks before overwriting (it would wait on stdin)
|
||||
subprocess.run(["ssh-keygen", "-q", "-Y", "sign", "-f", self.key, "-n", ns, bp], check=True, stdin=subprocess.DEVNULL, timeout=30)
|
||||
return open(bp + ".sig").read()
|
||||
|
||||
def test_good_signature_verifies(self):
|
||||
blob = b'{"op":"os_docker_step"}'
|
||||
self.assertEqual(osapply.Runner().verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob, self.sign(blob)), 0)
|
||||
|
||||
def test_changed_blob_wrong_namespace_or_wrong_principal_fail(self):
|
||||
blob = b'{"op":"os_docker_step"}'
|
||||
sig = self.sign(blob)
|
||||
r = osapply.Runner()
|
||||
self.assertNotEqual(r.verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob + b" ", sig), 0)
|
||||
self.assertNotEqual(r.verify_sig(self.signers, "someone-else", "felhom-op-v1", blob, sig), 0)
|
||||
self.assertNotEqual(r.verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob, self.sign(blob, ns="other-ns")), 0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user