From cc63dd5204e5082d170e36b33ba27e51a5aaf53a Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sat, 14 Nov 2026 12:11:06 +0100 Subject: [PATCH] Old kernels removed by the box (R-1368, wrapper mode kernel-cleanup); security profiles refreshed after a kernel change (R-1388, mode profile-refresh); Go 1.27.4 (R-1391) --- REUSE.md | 2 + cmd/felhom-agent/main.go | 32 +- configs/felhom-os-apply | 371 +++++++++++++- configs/test_felhom_os_apply.py | 602 +++++++++++++++++++++++ go.mod | 2 +- internal/osupdate/kernel.go | 128 +++++ internal/osupdate/kernel_test.go | 186 ++++++- internal/osupdate/leg.go | 18 +- internal/osupdate/leg_test.go | 13 +- internal/osupdate/profilerefresh.go | 101 ++++ internal/osupdate/profilerefresh_test.go | 97 ++++ 11 files changed, 1542 insertions(+), 10 deletions(-) create mode 100644 internal/osupdate/profilerefresh.go create mode 100644 internal/osupdate/profilerefresh_test.go diff --git a/REUSE.md b/REUSE.md index 04be35b..2fcaaee 100644 --- a/REUSE.md +++ b/REUSE.md @@ -98,6 +98,8 @@ |---|---|---|---|---| | `pvegate.Write` / `pvegate.Step` | internal/pvegate/pvegate.go | `Write(ctx) (release, waited, err)` / `Step(ctx) (end, err)` | R-812 option A: keep the agent's own /etc/pve writes out of a Proxmox package step (pmxcfs restarts) | Already wired at the two chokepoints — `Client.doBody` (every non-GET) and `ExecRunner.RunStdin` (`WritesEtcPVE`: pct config verbs, pvesm, pveum, felhom-pbs-apply create/reconcile). A new root CLI that writes /etc/pve goes into `WritesEtcPVE`, never its own lock. Never take `Step` around anything but the wrapper call (`Leg.runPVE`) — a `Write` inside a `Step` deadlocks until its context ends. | | `osupdate.KernelVerdict` / `Leg.KernelAfterBoot` | internal/osupdate/kernel.go | `KernelVerdict(before, after, tunnel, hubReached) (ok, why)` | R-836: THE one-shot-boot rule — the host health rule (`HostHealthVerdict`) AND the box reached the hub since the boot | The only judge of a new kernel. Never reboot the host from Go: every reboot is the wrapper's (`kernel-reboot`, `kernel-revert`), and only for a staged step. A new kernel-lane state lives in the wrapper's `/var/lib/felhom-kernel/state.json`, never in the agent's own files (the agent can write those). | +| `Leg.runKernelCleanup` / `kernelCleanupSkip` / `Leg.KernelCleanupDryRun` + the wrapper's `Kernel.cleanup` | internal/osupdate/kernel.go; configs/felhom-os-apply (mode `kernel-cleanup`) | `kernelCleanupSkip(appliance, trigger, host, pve, kernel, blk, okStep) string` ("" = call it) | R-1368 (`11` §5.11.1): old kernel images are removed by the WRAPPER's rule; the agent decides only WHEN (the night leg, after a healthy host step, never in a night with a kernel step) | The plan names NO package (C1). Never `apt-get autoremove`. Whether a box may do it is the root-owned ring-0 mark, never the hub's ring. The hub report carries NO kernel view and no kernel version (the hub would read it as a kernel step). `--selftest=kernel-cleanup-dry-run` changes nothing. | +| `Leg.ProfileRefresh` + the wrapper's `Apply.profile_refresh` | internal/osupdate/profilerefresh.go; configs/felhom-os-apply (mode `profile-refresh`) | `ProfileRefresh(ctx, why, gate, tries, wait) string` (the outcome) | R-1388 (`11` §5.12): load every AppArmor profile from a fresh compile once per kernel the box starts on; restart only a FAILED service whose own program's cached profile was stale | The plan names nothing (F1). A stale copy is told by the CACHED FILE changing at the rewrite — never by the kernel's loaded hash (it differs between a cache load and a fresh load for multi-profile files; `lxc-containers` never compiles the same twice). Called at agent start and after a bundle install, under the heavy-op gate. | | `Client.WaitTask` | internal/proxmox/task.go | `WaitTask(ctx, upid, opts) (TaskStatus, error)` | asserting EVERY mutating op | POST 200 ≠ success; authz can fail at task exec; `AllowWarnings` opt-in | | `Client.Pool` | internal/proxmox/query.go | `Pool(ctx, name) (PoolInfo, error)` | felhom-pool membership (the ownership registry, A1) | Needs `Pool.Audit` at `/pool/` (host-install v1.9.0+); `Pool.Allocate` does NOT satisfy the read; members can be storages (type `storage`, vmid 0) — filter them | | `Client` mutate wrappers (`RestoreLXC/Vzdump/DestroyLXC/Snapshot/Rollback/SetConfig/ResizeLXC/Start/Stop`) | internal/proxmox/mutate.go | return `(upid, error)` | all API mutations | Async → always pair with WaitTask; route via gate/queue, not ad-hoc | diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 1688416..7ddb961 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -251,6 +251,8 @@ func main() { os.Exit(runSelftestOSUpdate(context.Background(), cfg, logger, vmid)) case "os-facts": os.Exit(runSelftestFacts(context.Background(), cfg, logger, vmid)) + case "kernel-cleanup-dry-run": + os.Exit(runSelftestKernelCleanupDryRun(context.Background(), cfg, logger, vmid)) case "live-restore": os.Exit(runSelftestLiveRestore(context.Background(), cfg, logger, vmid)) case "pbs-verify": @@ -1284,6 +1286,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int } return release, nil }} + daemonCtx := ctx + profileGate := func() (func(), string, bool) { return heavyOps.TryAcquire("os-profile-refresh") } // Agent v0.143.0 (R-840): the config bundle — the box's root-owned files by a signed job; the wrapper verifies it. bundleExec := osupdate.ConfigUpdateExecutor{Leg: osLeg, URLTemplate: suCfg.URLTemplate, Username: suCfg.Username, Token: suCfg.Token, // The capability probe confirms from the agent's side: `sudo -l` lists every command the new sudoers grants. @@ -1294,7 +1298,13 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int names = append(names, d.Name) } logger.Warn("osupdate: capability probe after the config bundle", "ok", ok, "total", total, "degraded", strings.Join(names, ",")) + // R-1388: a new wrapper may know the profile refresh the old one refused — ask again (its own goroutine: + // it may wait for the heavy-op gate, and the signed job must not wait with it). + go osLeg.ProfileRefresh(daemonCtx, "config bundle installed", profileGate, 12, 5*time.Minute) }} + // R-1388 / R-1301 (`11` §5.12): after a start, load the security profiles from a fresh compile when the box runs + // another kernel than at the last refresh (the wrapper keeps that record and answers "not needed" otherwise). + go osLeg.ProfileRefresh(daemonCtx, "agent start", profileGate, 12, 5*time.Minute) // R-1285: the NEW agent runs the bundle policy pass once after its self-update commits — in the normal two-job // delivery the bundle was installed by the OLD process, which has no such pass (measured 2026-10-26). wireBundlePolicyAfterCommit(selfUpdateMgr, bundleExec, logger, version) @@ -3928,10 +3938,12 @@ func (f *selftestFlag) Set(v string) error { f.mode = "os-update" case "os-facts", "live-restore": // agent v0.142.0 f.mode = v + case "kernel-cleanup-dry-run": // R-1368: what the clean-up of old kernels WOULD do; changes nothing + f.mode = v case "wgtunnel": // dispatched since S3 but refused here until 2026-10-04 (TestSelftestFlag_AcceptsEveryDispatchedMode) f.mode = "wgtunnel" default: - return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|lanresolver|wgtunnel|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap|os-update|os-facts|live-restore)", v) + return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|lanresolver|wgtunnel|kernel-cleanup-dry-run|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap|os-update|os-facts|live-restore)", v) } return nil } @@ -4052,6 +4064,24 @@ func runSelftestFacts(ctx context.Context, cfg config.Config, logger *slog.Logge return 0 } +// runSelftestKernelCleanupDryRun asks the root wrapper what the clean-up of old kernels WOULD do on this box (R-1368, +// `11` §5.11.1): the keep set, the removal set, and the refusal or "not run" reason. It changes nothing. +// +// sudo -u felhom-agent felhom-agent --config … --selftest=kernel-cleanup-dry-run +func runSelftestKernelCleanupDryRun(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int { + px, _ := newProxmoxClient(cfg) + leg := newOSLeg(cfg, nil, px, logger) + raw, err := leg.KernelCleanupDryRun(ctx, vmid) + if err != nil { + fmt.Fprintln(os.Stderr, "selftest=kernel-cleanup-dry-run:", err) + return 1 + } + var v any + _ = json.Unmarshal(raw, &v) + printJSON("kernel clean-up — DRY RUN (nothing was changed)", v) + return 0 +} + // runSelftestLiveRestore is the ONE-TIME live-restore step (`09` decision 87) as a debug action — the night leg does // the same before a ring-0 Docker step. It prints the container ids before and after (they must not change). // diff --git a/configs/felhom-os-apply b/configs/felhom-os-apply index 67cc400..b5983e3 100755 --- a/configs/felhom-os-apply +++ b/configs/felhom-os-apply @@ -155,9 +155,28 @@ KERNEL_IMAGE_RE = re.compile(r"^proxmox-kernel-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pv # helper and the firmware the kernel loads. Everything else in HOST_SLOW_RE (grub, shim, microcode, efibootmgr) stays out. KERNEL_UPGRADE_RE = re.compile(r"^(proxmox-default-kernel|proxmox-kernel-[0-9]+\.[0-9]+|proxmox-kernel-helper|pve-firmware)$") KERNEL_MIN_GAP = 20 * 3600 # never two kernel steps in one night -KERNEL_MODES = ("kernel-status", "kernel-reboot", "kernel-boot", "kernel-good", "kernel-revert", "kernel-cancel") +KERNEL_MODES = ("kernel-status", "kernel-reboot", "kernel-boot", "kernel-good", "kernel-revert", "kernel-cancel", + "kernel-cleanup") # phases: staged → oneshot → judging → good | reverting → self_reverted | revert_failed; oneshot → fell_back; staged → cancelled KERNEL_ACTIVE = ("staged", "oneshot", "judging", "reverting") +# The clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1). A step that ENDED in one of these phases +# and is not "good" keeps every kernel (the box fell back or reverted); any other phase is "not ended" (P1). +# The security-profile refresh (R-1388, R-1301, `11` §5.12; measured on demo-hp 2026-11-14, +# felhom.eu audits/profile-refresh-spike-2026-11-14/). AppArmor keeps one cache folder per kernel feature set; a +# profile that changed while ANOTHER kernel ran leaves an old compiled copy in this kernel's folder, with a time that +# looks newer than its source — and the kernel loads it at the next start on this kernel. +PROFILE_DIR = "/etc/apparmor.d" +PROFILE_CACHE = "/var/cache/apparmor" +PROFILE_STATE = "/var/lib/felhom-os-apply/profile-refresh.json" # 0644 root: the kernel the last refresh ran on +PROFILE_REFRESH_CMD = ["apparmor_parser", "--replace", "--skip-read-cache", "--write-cache", "--", PROFILE_DIR] +PROFILE_PATH_NAME_RE = re.compile(r"^/[A-Za-z0-9._/+-]+$") # a profile whose name is one plain program path +UNIT_NAME_RE = re.compile(r"^[A-Za-z0-9@:._-]+\.service$") +PROFILE_RESTART_MAX = 3 +KERNEL_ENDED_NOT_GOOD = ("self_reverted", "revert_failed", "fell_back", "cancelled") +KERNEL_LEFTOVER_PATHS = ("/lib/modules/{v}", "/boot/vmlinuz-{v}", "/boot/initrd.img-{v}", "/boot/System.map-{v}", + "/boot/config-{v}") +PINNED_KVER_RE = re.compile(r"gnulinux-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve)-advanced-") +ANY_KVER_RE = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve") # ---------- the config bundle (R-840, agent v0.143.0, `11` §5.4.2) ---------- # A signed `agent_config_update` job carries {agent_version, bundle_sha256}; the bundle is ONE JSON file built from this @@ -495,7 +514,7 @@ class Apply: def check_plan(self, plan): mode = plan.get("mode", "apply") if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "bundle-policy", "agent_update", - "oom-check") \ + "oom-check", "profile-refresh") \ + KERNEL_MODES: raise Refused("R11", f"unknown mode {mode!r}") if mode == "agent_update": @@ -510,6 +529,16 @@ class Apply: if plan.get("layer") != "host": raise Refused("R11", "bundle-policy is a host-layer mode") return mode, "host", 0, "bundle-policy" + if mode == "profile-refresh": + if plan.get("layer") != "host": + raise Refused("R11", "profile-refresh is a host-layer mode") + # it takes NOTHING from the plan but dry_run: no profile name, no unit name, no package (R-1388) + for k in ("packages", "allow_new", "signed", "select", "undo", "snapshot", "profiles", "units", "force"): + if plan.get(k): + raise Refused("F1", f"a profile-refresh plan carries {k!r} — it takes nothing from a plan but dry_run") + if not isinstance(plan.get("dry_run", False), bool): + raise Refused("F1", "dry_run must be true or false") + return mode, "host", 0, "profile-refresh" layer = plan.get("layer") if layer not in ("guest", "host", "docker", "pve", "kernel"): raise Refused("R12", f"layer {layer!r} is not guest, host, docker, pve or kernel") @@ -540,6 +569,17 @@ class Apply: rid = plan.get("release_id", "") if not isinstance(rid, str) or not re.match(r"^[A-Za-z0-9._:-]{1,80}$", rid): raise Refused("R11", f"release_id {rid!r} is not a plain id") + if mode == "kernel-cleanup": + # `11` §5.11.1, rule 4 of the release: the clean-up NEVER takes a package name (or any choice of what to + # remove) from a plan — the wrapper computes the set from the box. Pinned by test_cleanup_plan_names_refused. + for k in ("packages", "allow_new", "signed", "expect_kver", "undo", "snapshot"): + if plan.get(k): + raise Refused("C1", f"a kernel-cleanup plan carries {k!r} — the clean-up takes nothing from a plan " + f"but dry_run; the wrapper computes the set from the box") + if plan.get("select", "listed") != "listed": + raise Refused("C1", "a kernel-cleanup plan carries a select — the wrapper computes the set from the box") + if not isinstance(plan.get("dry_run", False), bool): + raise Refused("C1", "dry_run must be true or false") if plan.get("allow_new"): raise Refused("R6", "allow_new is a slow-lane field; the fast lane never adds a package") select = plan.get("select", "listed") @@ -1084,6 +1124,9 @@ class Apply: return Bundle(self).from_plan(plan) if self.mode == "bundle-policy": return Bundle(self).policy_pass() + if self.mode == "profile-refresh": + self.check_appliance() + return self.profile_refresh(plan) if self.mode == "agent_update": return self.agent_update(plan) if self.layer in ("host", "pve", "kernel"): @@ -1149,6 +1192,107 @@ class Apply: self.report["oom_check"] = self.oom_check() return 0 + # ---------- the security-profile refresh (R-1388, R-1301, `11` §5.12) ---------- + def profile_cache(self): + """{"/": sha256} of every cached profile, every kernel's folder.""" + out = {} + for d in self.r.list_dir(PROFILE_CACHE): + dp = f"{PROFILE_CACHE}/{d}" + if not self.r.isdir(dp): + continue + for f in self.r.list_dir(dp): + if f.startswith("."): + continue + try: + out[f"{d}/{f}"] = sha256_hex(self.r.read_bytes(f"{dp}/{f}")) + except OSError: + out[f"{d}/{f}"] = "unreadable" + return out + + def profile_refresh(self, plan): + """Load every AppArmor profile from a FRESH compile (the cache ignored and rewritten), once per kernel the box + starts on; then restart ONLY a failed service whose own program's profile had a stale cached copy. + + Changes no package and takes no name from the plan. Measured on demo-hp with both guests running: 0.5 s, no + guest or container touched, the same profiles loaded in the same modes. + + WHY NOT a "loaded hash changed" check: for a file that defines several profiles the kernel's hash differs + between a cache load and a fresh load of the SAME policy (17 of 119 on demo-hp), and one file (lxc-containers) + compiles to a different binary every time. The only reliable sign of a stale copy is the CACHED FILE changing + when it is rewritten — that is what the restart rule reads.""" + r, dry = self.r, plan.get("dry_run") is True + res = {"dry_run": dry, "outcome": None, "rewritten": [], "restarted": [], "not_restarted": []} + self.report["profile_refresh"] = res + rc, out, _ = r.host(["uname", "-r"], 30) + kernel = out.strip() if rc == 0 else "" + try: + rec = json.loads(r.read_file(PROFILE_STATE)) + rec = rec if isinstance(rec, dict) else {} + except (OSError, ValueError): + rec = {} + res["kernel"], res["last_refresh_kernel"], res["last_refresh_at"] = kernel or "unknown", rec.get("kernel"), rec.get("at") + if not kernel: + raise Refused("F2", "the running kernel could not be read") + if not r.isdir(PROFILE_DIR) or not r.isdir(PROFILE_CACHE): + res["outcome"] = "not_needed" + res["reason"] = "this box has no AppArmor profile folder or no profile cache" + return 0 + needed = rec.get("kernel") != kernel + if not needed: + res["outcome"], res["reason"] = "not_needed", f"the profiles were refreshed on this kernel ({kernel}) at {rec.get('at')}" + return 0 + if dry: + res["outcome"], res["would"] = "dry_run", "refresh" + return 0 + before = self.profile_cache() + t0 = time.time() + rc, out, err = r.host(list(PROFILE_REFRESH_CMD), 600) + res["seconds"] = round(time.time() - t0, 1) + if rc != 0: + res["outcome"] = "failed" + self.report["failed"] = {"rc": rc, "step": "profile-refresh", "tail": (out + err).strip().splitlines()[-3:]} + r.log(f"os-apply: FAILED rc={rc} step=profile-refresh (no record written: the next start tries again)") + return 3 + after = self.profile_cache() + res["rewritten"] = sorted(k for k in after if before.get(k) != after[k]) + # --- the narrow restart rule: a FAILED service, whose ExecStart program IS the name of a profile whose cached + # copy was stale. Never "whatever failed"; never a unit named by the plan. --- + programs = {} + for key in res["rewritten"]: + fname = key.split("/", 1)[1] + rc_n, names, _ = r.host(["apparmor_parser", "-N", f"{PROFILE_DIR}/{fname}"], 60) + if rc_n != 0: + continue + for n in names.split(): + if PROFILE_PATH_NAME_RE.match(n): + programs[n] = fname + if programs: + rc_u, units, _ = r.host(["systemctl", "list-units", "--type=service", "--state=failed", "--no-legend", "--plain"], 60) + for line in (units.splitlines() if rc_u == 0 else []): + unit = (line.split() or [""])[0] + if not UNIT_NAME_RE.match(unit): + continue + _, es, _ = r.host(["systemctl", "show", "-p", "ExecStart", "--value", unit], 30) + m = re.search(r"\bpath=(\S+)", es) + prog = m.group(1) if m else "" + if prog not in programs: + continue + if len(res["restarted"]) >= PROFILE_RESTART_MAX: + res["not_restarted"].append({"unit": unit, "program": prog, "reason": f"more than {PROFILE_RESTART_MAX} units match — look at the box"}) + continue + rc_r, o2, e2 = r.host(["systemctl", "restart", unit], 120) + _, act, _ = r.host(["systemctl", "is-active", unit], 30) + res["restarted"].append({"unit": unit, "program": prog, "profile_file": programs[prog], "rc": rc_r, + "active": act.strip() == "active"}) + r.log(f"os-apply: PROFILE REFRESH restarted {unit} (failed; its program {prog} had a stale cached profile) " + f"rc={rc_r} now={act.strip() or 'unknown'}") + r.put_file(PROFILE_STATE, (json.dumps({"kernel": kernel, "at": _iso(r.now()), "rewritten": res["rewritten"], + "restarted": [x["unit"] for x in res["restarted"]]}, indent=2, sort_keys=True) + "\n").encode(), 0o644) + res["outcome"] = "refreshed" + r.log(f"os-apply: PROFILE REFRESH DONE kernel={kernel} (last {rec.get('kernel') or 'never'}) seconds={res['seconds']} " + f"rewritten={len(res['rewritten'])} ({', '.join(res['rewritten'][:8])}) restarted={[x['unit'] for x in res['restarted']]}") + return 0 + # ---------- R-528: the memory-kill check ---------- def oom_check(self): """Run one throwaway container over its memory cap in the guest and read what the engine says about it. @@ -1493,6 +1637,11 @@ class Kernel: kernel-good the one-shot boot was healthy: the new kernel becomes the GRUB default. kernel-revert the one-shot boot was NOT healthy: reboot ONCE into the old kernel (still the default). kernel-cancel drop a staged step: clear the flag (the package stays installed, the default never moved). + kernel-cleanup remove old kernel images by the rule of `11` §5.11.1 (R-1368, decision 286). The plan names nothing + (C1); `dry_run: true` computes and reports, and changes nothing. Its own refusals: P1–P3, K1, N1, + S1, S2 (the design's table) and C1 (the plan names something). "Not run" (exit 0, nothing + changed): no ring-0 mark, no kernel step yet, the newest step did not end good, the box does not + run its newest kernel, inside a step's 20 h window. kernel-status read only.""" def __init__(self, apply, plan): @@ -1646,7 +1795,8 @@ class Kernel: self.report["kernel"] = v return 0 fn = {"apply": self.stage, "kernel-reboot": self.reboot_staged, "kernel-boot": self.after_boot, - "kernel-good": self.good, "kernel-revert": self.revert, "kernel-cancel": self.cancel}[mode] + "kernel-good": self.good, "kernel-revert": self.revert, "kernel-cancel": self.cancel, + "kernel-cleanup": self.cleanup}[mode] rc = fn() self.report["kernel"] = self.view() return rc @@ -1882,6 +2032,221 @@ class Kernel: self.log(f"os-apply: KERNEL CANCELLED {st.get('to')} (installed, never the default; flag cleared)") return 0 + # ---------- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ---------- + def kernel_images(self): + """kver -> [installed image package names] (dpkg status ii only).""" + out = {} + for name in sorted(self.a.installed()): + m = KERNEL_IMAGE_RE.match(name) + if m: + out.setdefault(m.group(1), []).append(name) + return out + + def kver_newest(self, kvers): + best = None + for k in kvers: + if best is None or self.a.dpkg_cmp(k[:-4], "gt", best[:-4]): + best = k + return best + + def read_or_none(self, path): + try: + return self.r.read_file(path) + except OSError: + return None + + def cleanup_sources(self): + """Every place that names a kernel the lane still needs, read from the box ONCE: (kver, why) pairs. This is + the keep set's only input, and K1's only input — K1 reads it again by itself (cleanup_protected).""" + st = self.state() + cfg = self.grub_cfg() + dflt = self.default_kver(cfg) + m = PINNED_KVER_RE.search(self.read_or_none(KERNEL_DEFAULT_CFG) or "") + pinned = m.group(1) if m else None + flag = self.flag() or "" + m = ANY_KVER_RE.search(flag) + m2 = ANY_KVER_RE.search(self.read_or_none(KERNEL_PIN_FILE) or "") + src = [(self.running(), "running"), + (st.get("from"), "the kernel step's `from` (the last kernel that booted well; a self-revert boots it)"), + (st.get("to"), "the kernel step's `to`"), + (st.get("default_before"), "the kernel step's `default_before`"), + (dflt if KVER_RE.match(dflt) else None, "the default in grub.cfg"), + (pinned, "the pinned default file"), + (m.group(0) if m else None, "the one-shot flag"), + (m2.group(0) if m2 else None, "an operator's boot pin")] + return st, dflt, pinned, flag, [(k, why) for k, why in src if isinstance(k, str) and KVER_RE.match(k)] + + def cleanup_keep(self, images, sources): + """The keep set: every kernel a source names, and the newest installed. (cleanup() adds the next lower + installed kernel when no step record names the last good one.)""" + keep = {} + for k, why in sources: + keep.setdefault(k, []).append(why) + newest = self.kver_newest(images) + if newest: + keep.setdefault(newest, []).append("newest installed") + return keep + + def cleanup_protected(self): + """K1's own reading (`11` §5.11.1: "K1 is the rule's own check and must stay its own"): the kernels that must + never reach the purge line, read from the box AGAIN — not taken from the keep set that built the removal set.""" + _, _, _, _, sources = self.cleanup_sources() + return {k: why for k, why in sources} + + def cleanup_simulate(self, names): + """apt's simulation of exactly `names`: (rc, removes, installs, last line).""" + rc, out, err = self.a.x(APT_ENV + ["apt-get", "-s", "-q", "purge"] + names) + removes = sorted({l.split()[1] for l in out.splitlines() if l.startswith(("Remv ", "Purg ")) and len(l.split()) > 1}) + installs = sorted({l.split()[1] for l in out.splitlines() if l.startswith(("Inst ", "Conf ")) and len(l.split()) > 1}) + return rc, removes, installs, ((err or out).strip().splitlines() or ["?"])[-1][:200] + + def cleanup(self): + a = self.a + dry = self.plan.get("dry_run") is True + res = {"dry_run": dry, "outcome": None, "not_run": [], "refusals": [], "removed": [], "leftovers": []} + self.report["kernel_cleanup"] = res + not_run, refuse = res["not_run"], res["refusals"] + # --- may this box run the clean-up at all? (ring 0 only in this release; the mark is ROOT-owned) --- + try: + ring0 = a.load_trust().get("ring0_slow_lane") is True + except Refused as e: + ring0 = False + not_run.append({"code": "no-ring0-mark", "reason": f"no usable trust record: {e.reason}"}) + if not ring0 and not not_run: + not_run.append({"code": "no-ring0-mark", "reason": "this box has no root-owned ring-0 mark — the clean-up runs on " + "ring 0 only in this release"}) + if a.apt_lock_held(): + raise Refused("R9", "another apt/dpkg holds the lock on the host") + if not dry and not not_run: + a.repair() # repair first (`11` §5.4.1): a purge a power cut interrupted is made whole, THEN the rule is computed + # --- read the box --- + images = self.kernel_images() + running = self.running() + newest = self.kver_newest(images) + raw_state = self.read_or_none(KERNEL_STATE) + st, dflt, pinned, flag, sources = self.cleanup_sources() + phase = st.get("phase", "none") if raw_state is not None else "none" + res.update(running=running or "unknown", newest=newest, phase=phase, default=dflt, pinned_default=pinned, + installed={k: images[k] for k in sorted(images, key=lambda v: [int(x) for x in re.findall(r"\d+", v)])}) + # --- "when it may run" (decision 286): not run, nothing changed --- + if raw_state is None or not st: + not_run.append({"code": "no-step", "reason": "no kernel step yet (no state file) — nothing proves which kernel booted well"}) + elif phase in KERNEL_ENDED_NOT_GOOD: + not_run.append({"code": "step-not-good", "reason": f"the newest kernel step ended {phase!r}, not good — every kernel stays"}) + elif phase != "good": + refuse.append({"code": "P1", "reason": f"a kernel step is in phase {phase!r} — not ended (or not a phase this wrapper knows)"}) + if running and newest and running in images and running != newest: + not_run.append({"code": "not-on-newest", "reason": f"the box runs {running}, not its newest installed kernel {newest} — every kernel stays"}) + times = [x for x in (_parse_iso(st.get(k)) for k in ("staged_at", "rebooted_at", "result_at")) if x is not None] + if times and self.r.now() - max(times) < KERNEL_MIN_GAP: + not_run.append({"code": "window", "reason": "a kernel step staged, booted or ended within the last 20 hours — never in its night"}) + # --- the refusal table --- + if flag: + refuse.append({"code": "P2", "reason": f"the one-shot flag names {flag!r}"}) + if pinned is None or pinned != dflt or pinned not in images: + refuse.append({"code": "P3", "reason": f"the pinned default {pinned!r} and grub.cfg's default {dflt!r} do not name the same installed kernel"}) + if not running or running not in images: + refuse.append({"code": "K1", "reason": f"the running kernel {running or 'unknown'!r} is not an installed kernel image package"}) + # --- the keep set and the removal set (exact package names) --- + keep = self.cleanup_keep(images, sources) + if running and not (st.get("to") == running and st.get("from") in images): + lower = [k for k in images if a.dpkg_cmp(k[:-4], "lt", running[:-4])] if running in images else [] + lg = self.kver_newest(lower) + if lg: + keep.setdefault(lg, []).append("the next lower installed kernel (no step record names the last good one)") + removal = sorted(n for k in images if k not in keep for n in images[k]) + res["kept"] = {k: keep[k] for k in sorted(keep) if k in images} + res["removal"] = removal + # K1 — its own reading of the box, against the removal set + by_name = {n: k for k, names in images.items() for n in names} + protected = self.cleanup_protected() + for n in removal: + if n not in by_name: + refuse.append({"code": "N1", "reason": f"{n} is not an installed kernel image package"}) + elif by_name[n] in protected or by_name[n] == newest: + refuse.append({"code": "K1", "reason": f"{n} is a keep-kernel ({protected.get(by_name[n], 'newest installed')})"}) + # S1 / S2 — apt's simulation of exactly that set + if removal and not [x for x in refuse if x["code"] in ("K1", "N1")]: + rc, removes, installs, tail = self.cleanup_simulate(removal) + res["simulation"] = {"rc": rc, "removes": removes, "installs": installs} + if rc != 0: + refuse.append({"code": "S1", "reason": "the simulation failed: " + tail}) + if installs: + refuse.append({"code": "S1", "reason": f"the simulation would install or configure {', '.join(installs[:6])}"}) + extra = [p for p in removes if p not in removal] + if extra: + refuse.append({"code": "S2", "reason": f"the simulation would also remove {', '.join(extra[:8])} — not in the removal set"}) + missing = [p for p in removal if p not in removes] + if missing and rc == 0: + refuse.append({"code": "S1", "reason": f"the simulation does not remove {', '.join(missing[:6])}"}) + self.log(f"os-apply: KERNEL CLEANUP {'DRY RUN ' if dry else ''}running={running or 'unknown'} newest={newest} phase={phase} " + f"installed={len(images)} keep={len(res['kept'])} removal={len(removal)} not_run={[x['code'] for x in not_run]} " + f"refusals={[x['code'] for x in refuse]}") + # --- the verdict --- + if dry: + res["outcome"] = "dry_run" + res["would"] = "not_run" if not_run else "refuse" if refuse else "remove" if removal else "nothing" + return 0 + if not_run: + res["outcome"] = "not_run" + return 0 + if refuse: + raise Refused(refuse[0]["code"], "; ".join(f"{x['code']} {x['reason']}" for x in refuse)[:600]) + if not removal: + res["outcome"] = "nothing" + self.log("os-apply: KERNEL CLEANUP nothing to remove") + return 0 + # --- the purge: ONE apt-get purge of the exact names; then prove the boot setup did not move --- + gens_before = {p: (self.r.lexists(p), self.read_or_none(p)) for p in (ONESHOT_SNIPPET, ONESHOT_ENTRIES)} + pin_before = self.read_or_none(KERNEL_DEFAULT_CFG) + free_before = a.free_bytes() + t0 = time.time() + rc, out, err = a.x(APT_ENV + ["apt-get", "-y", "-q", "purge"] + removal) + res["seconds"] = round(time.time() - t0, 1) + if rc != 0: + _, aud, _ = a.x(["dpkg", "--audit"]) + res["outcome"] = "failed" + self.report["failed"] = {"rc": rc, "step": "purge", "dpkg_audit": (aud.strip().splitlines() or ["clean"])[0], + "tail": (out + err).strip().splitlines()[-3:]} + self.log(f"os-apply: FAILED rc={rc} step=purge (the next run repairs dpkg first, then computes the rule again)") + return 3 + after = self.kernel_images() + after_names = {n for names in after.values() for n in names} + res["removed"] = [n for n in removal if n not in after_names] + free_after = a.free_bytes() + res["bytes_gained"] = free_after - free_before if free_before >= 0 and free_after >= 0 else None + problems = [] + if len(res["removed"]) != len(removal): + problems.append(f"still installed after the purge: {', '.join(n for n in removal if n in after_names)}") + gone = [k for k in res["kept"] if k not in after] + if gone: + problems.append(f"a KEPT kernel is no longer installed: {', '.join(gone)}") + got = self.default_kver(self.grub_cfg()) + if got != dflt: + problems.append(f"the default in grub.cfg moved from {dflt} to {got}") + if self.read_or_none(KERNEL_DEFAULT_CFG) != pin_before: + problems.append("the pinned default file changed") + if self.read_or_none(KERNEL_STATE) != raw_state: + problems.append("the kernel step's state file changed") + for p, (was, body) in gens_before.items(): + if self.r.lexists(p) != was or self.read_or_none(p) != body: + problems.append(f"{p} changed") + for k in sorted(set(by_name[n] for n in res["removed"])): + if k in after: + continue # another image package of the same kernel is still installed + res["leftovers"] += [p.format(v=k) for p in KERNEL_LEFTOVER_PATHS if self.r.lexists(p.format(v=k))] + if problems: + # reported LOUDLY, never repaired by hand here (`11` §5.11.1): the operator looks at the box + res["outcome"] = "failed" + self.report["failed"] = {"rc": 3, "step": "verify", "reason": "; ".join(problems)[:600]} + self.log("os-apply: FAILED rc=3 step=verify after the kernel purge — " + "; ".join(problems)) + return 3 + res["outcome"] = "removed" + self.log(f"os-apply: KERNEL CLEANUP DONE removed={len(res['removed'])} ({', '.join(res['removed'])}) " + f"kept={', '.join(res['kept'])} gained={res['bytes_gained']} B seconds={res['seconds']} " + f"leftovers={len(res['leftovers'])}; default {dflt} (proved from grub.cfg)") + return 0 + class Bundle: """The config bundle (R-840, `11` §5.4.2): every root-owned file the installer's step 5 writes, installed as ONE diff --git a/configs/test_felhom_os_apply.py b/configs/test_felhom_os_apply.py index b27de26..f70a4be 100644 --- a/configs/test_felhom_os_apply.py +++ b/configs/test_felhom_os_apply.py @@ -2031,5 +2031,607 @@ class KernelLane(unittest.TestCase): self.assertIn("one-shot", h["kernel_next_boot_source"]) + +# ---------- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ---------- +K = ["7.0.2-6-pve", "7.0.14-20-pve", "7.0.14-22-pve", "7.0.14-26-pve", "7.0.14-28-pve"] # five installed kernels +C_FROM, C_RUN = "7.0.14-26-pve", "7.0.14-28-pve" +GB = 1024 ** 3 + + +def img(k): + return f"proxmox-kernel-{k}-signed" + + +class CFake(KFake): + """A ring-0 appliance after a good kernel step 26 -> 28: five kernels installed, the box runs its newest, the + default (grub.cfg and the pinned file) is the running kernel, the flag is empty. `apt-get purge` is emulated like + the real one: the image leaves dpkg and /boot, and the kernel's postrm runs update-grub.""" + + def __init__(self): + super().__init__() + self.running = C_RUN + self.boot = set(K) + self.modules = set(K) # /lib/modules/ + for n in [x for x in self.installed if osapply.KERNEL_IMAGE_RE.match(x)]: + del self.installed[n] + for k in K: + self.installed[img(k)] = k[:-4] + self.installed["proxmox-kernel-7.0"] = C_RUN[:-4] + self.env = {"felhom_next": ""} + self.tree[osapply.KERNEL_DEFAULT_CFG] = (f'GRUB_DEFAULT="gnulinux-advanced-{U}>gnulinux-{C_RUN}-advanced-{U}"\n') + self.state = {"phase": "good", "step_id": "ring0-x", "from": C_FROM, "to": C_RUN, "default_before": C_FROM, + "vmid": 9201, "staged_at": "2026-10-01T02:50:00Z", "rebooted_at": "2026-10-01T02:51:00Z", + "result_at": "2026-10-01T03:10:00Z", "self_revert_used": False} + self.write_state() + self.cfg = self.render() + self.purges, self.purge_rc, self.purge_moves_default, self.purge_extra_sim = [], 0, False, [] + self.plan = {"release_id": "kernel", "layer": "kernel", "lane": "slow", "vmid": 9201, "mode": "kernel-cleanup", + "packages": []} + + def write_state(self): + if self.state is None: + self.tree.pop(osapply.KERNEL_STATE, None) + else: + self.tree[osapply.KERNEL_STATE] = json.dumps(self.state) + + def lexists(self, p): + m = re.match(r"^/lib/modules/(.+)$", p) + if m: + return m.group(1) in self.modules + return super().lexists(p) + + def changing(self): + """Every command that changes the box (the purge, a repair, update-grub, a flag write, a file write).""" + out = [] + for c in self.calls: + a = [x for x in c[-1] if not re.match(r"^[A-Z_]+=", x) and x != "env"] + if (a[0] == "apt-get" and "-s" not in a and "--print-uris" not in a) or a[0] in ("update-grub", "dpkg-reconfigure") \ + or (a[0] == "dpkg" and a[1] == "--configure") or (a[0] == "grub-editenv" and a[2] != "list") \ + or (a[0] == "systemctl" and a[1] == "reboot"): + out.append(a) + return out + + def emulate(self, argv): + a = [x for x in argv if not re.match(r"^[A-Z_]+=", x) and x != "env"] + if a[0] == "apt-get" and "purge" in a and "-s" not in a: + names = [x for x in a[a.index("purge") + 1:] if not x.startswith("-")] + self.purges.append(names) + if self.purge_rc: + self.dpkg_journal = ["0001"] # a purge cut in the middle leaves dpkg's update journal + return self.purge_rc, "", "E: Sub-process /usr/bin/dpkg returned an error code (1)" + for n in names: + self.installed.pop(n, None) + m = osapply.KERNEL_IMAGE_RE.match(n) + if m: + self.boot.discard(m.group(1)) + self.free += GB + if self.purge_moves_default: + self.tree.pop(osapply.KERNEL_DEFAULT_CFG, None) + self.cfg = self.render() # the kernel's postrm runs update-grub (zz-update-grub) + return 0, "Removing proxmox-kernel ...\n", "" + return super().emulate(argv) + + def sim(self, a): + if "purge" in a: + names = [x for x in a[a.index("purge") + 1:] if not x.startswith("-")] + if getattr(self, "purge_sim_rc", 0): + return self.purge_sim_rc, "", "E: Unable to locate package" + out = "".join(f"Purg {n} [{self.installed[n]}]\n" for n in names if n in self.installed) + return 0, out + "".join(l + "\n" for l in self.purge_extra_sim), "" + return super().sim(a) + + +def cfake(**kw): + f = CFake() + for k, v in kw.items(): + setattr(f, k, v) + return f + + +def cstate(f, **kw): + f.state.update(kw) + f.write_state() + return f + + +class KernelCleanup(unittest.TestCase): + """§5 of the release-2a brief: one test per acceptance scenario, each on the stored effect (the exact apt line or + the absence of any changing command, the report, the exit code). Red-proofs (the four mutations the brief names): + felhom.eu audits/release-2a-2026-11-14/A/redproof.txt.""" + + REMOVE = [img(k) for k in ("7.0.14-20-pve", "7.0.14-22-pve", "7.0.2-6-pve")] # sorted by name + + def nothing_changed(self, f): + self.assertEqual(f.purges, [], "no purge") + self.assertEqual(f.changing(), [], "no command that changes the box") + self.assertEqual(sorted(f.boot), sorted(K), "every kernel is still there") + + def refused(self, f, code): + before = (f.tree.get(osapply.KERNEL_STATE), f.tree.get(osapply.KERNEL_DEFAULT_CFG), f.cfg, dict(f.env or {})) + rc, rep = run(f) + self.assertEqual(rc, 2, rep) + self.assertEqual(rep["refused"]["code"], code, rep) + self.nothing_changed(f) + self.assertEqual((f.tree.get(osapply.KERNEL_STATE), f.tree.get(osapply.KERNEL_DEFAULT_CFG), f.cfg, dict(f.env or {})), before) + return rep + + def not_run(self, f, code): + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + c = rep["kernel_cleanup"] + self.assertEqual(c["outcome"], "not_run", rep) + self.assertIn(code, [x["code"] for x in c["not_run"]], rep) + self.assertIsNone(rep.get("refused")) + self.nothing_changed(f) + return rep + + # --- scenario 1: the normal night --- + def test_s1_the_normal_night_purges_exactly_the_old_kernels_in_one_command(self): + f = cfake() + state0, pin0 = f.tree[osapply.KERNEL_STATE], f.tree[osapply.KERNEL_DEFAULT_CFG] + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertEqual(f.purges, [self.REMOVE], "ONE apt-get purge of the exact names") + purge = [c[-1] for c in f.calls if "purge" in c[-1] and "-s" not in c[-1]] + self.assertEqual(len(purge), 1) + self.assertEqual(purge[0][purge[0].index("apt-get"):], ["apt-get", "-y", "-q", "purge"] + self.REMOVE) + self.assertNotIn("autoremove", " ".join(" ".join(c[-1]) for c in f.calls), "never apt's own clean-up") + c = rep["kernel_cleanup"] + self.assertEqual(c["outcome"], "removed") + self.assertEqual(c["removed"], self.REMOVE) + self.assertEqual(sorted(c["kept"]), sorted([C_FROM, C_RUN])) + self.assertEqual(c["bytes_gained"], 3 * GB) + self.assertEqual(sorted(f.boot), sorted([C_FROM, C_RUN])) + # the default in grub.cfg, the pinned default file and the step's state file are unchanged + self.assertEqual(osapply.Kernel.default_kver(f.cfg), C_RUN) + self.assertEqual(f.tree[osapply.KERNEL_DEFAULT_CFG], pin0) + self.assertEqual(f.tree[osapply.KERNEL_STATE], state0) + self.assertEqual(f.env, {"felhom_next": ""}) + self.assertEqual(f.reboots, [], "the clean-up never reboots") + self.assertEqual(f.update_grubs, 0, "the wrapper itself runs no update-grub (the kernel's postrm did)") + # leftovers are reported, never deleted + self.assertEqual(c["leftovers"], [f"/lib/modules/{k}" for k in ("7.0.14-20-pve", "7.0.14-22-pve", "7.0.2-6-pve")]) + self.assertEqual(f.modules, set(K), "a left-over folder is reported, not deleted") + + # --- scenario 2: the second night --- + def test_s2_the_second_night_removes_nothing(self): + f = cfake() + self.assertEqual(run(f)[0], 0) + f.calls.clear() + f.purges.clear() + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertEqual(rep["kernel_cleanup"]["outcome"], "nothing") + self.assertEqual(rep["kernel_cleanup"]["removal"], []) + self.assertEqual(f.purges, []) + self.assertEqual(f.changing(), [], "no apt call that changes anything") + self.assertFalse([c for c in f.calls if "purge" in c[-1]], "not even a simulation: there is nothing to simulate") + + # --- scenario 3: a step is not over (P1) --- + def test_s3_a_step_that_is_not_over_refuses(self): + for phase in ("staged", "oneshot", "judging", "reverting", "some-new-phase"): + f = cstate(cfake(), phase=phase) + rep = self.refused(f, "P1") + self.assertIn(phase, rep["refused"]["reason"]) + + # --- scenario 4: the flag (P2); the pinned default against grub.cfg (P3) --- + def test_s4_the_flag_names_a_kernel_refuses(self): + f = cfake(env={"felhom_next": "7.0.14-26-pve"}) + self.refused(f, "P2") + + def test_s4_pinned_default_and_grub_cfg_disagree_refuses(self): + f = cfake() + f.tree[osapply.KERNEL_DEFAULT_CFG] = f'GRUB_DEFAULT="gnulinux-advanced-{U}>gnulinux-{C_FROM}-advanced-{U}"\n' + self.refused(f, "P3") # grub.cfg still says 28, the file says 26 + g = cfake() + del g.tree[osapply.KERNEL_DEFAULT_CFG] + self.refused(g, "P3") # no pinned file at all + h = cfake() # both name a kernel that is not installed + h.tree[osapply.KERNEL_DEFAULT_CFG] = f'GRUB_DEFAULT="gnulinux-advanced-{U}>gnulinux-7.0.14-99-pve-advanced-{U}"\n' + h.cfg = h.cfg.replace(f"gnulinux-{C_RUN}-advanced-{U}\"", f"gnulinux-7.0.14-99-pve-advanced-{U}\"", 1) + self.refused(h, "P3") + + # --- scenario 5: the box fell back / does not run its newest kernel --- + def test_s5_a_box_that_fell_back_keeps_every_kernel(self): + for phase in ("fell_back", "self_reverted", "revert_failed", "cancelled"): + self.not_run(cstate(cfake(running=C_FROM), phase=phase), "step-not-good") + + def test_s5_a_box_that_does_not_run_its_newest_kernel_keeps_every_kernel(self): + f = cfake(running=C_FROM) # the step says good, but the box runs the older kernel + self.not_run(f, "not-on-newest") + + # --- scenario 6: the wrong case (K1) — one case per kept-kernel kind --- + def k1(self, kind, prepare): + """The keep computation is BROKEN on purpose (it forgets `kind`); K1 reads the box by itself and must refuse + before any apt call.""" + f = cfake() + victim = prepare(f) + real = osapply.Kernel.cleanup_keep + + def broken(self_, images, sources): + keep = real(self_, images, [(k, why) for k, why in sources if k != victim]) + keep.pop(victim, None) # … also when it is the newest installed + return keep + osapply.Kernel.cleanup_keep = broken + try: + rc, rep = run(f) + finally: + osapply.Kernel.cleanup_keep = real + self.assertEqual(rc, 2, f"{kind}: {rep}") + self.assertEqual(rep["refused"]["code"], "K1", f"{kind}: {rep}") + self.assertIn(img(victim), rep["refused"]["reason"]) + self.assertFalse([c for c in f.calls if "purge" in c[-1]], f"{kind}: refused BEFORE any apt call, also the simulation") + self.nothing_changed(f) + + def test_s6_k1_the_running_kernel(self): + self.k1("running", lambda f: C_RUN) + + def test_s6_k1_the_steps_from(self): + self.k1("from", lambda f: C_FROM) + + def test_s6_k1_the_steps_to(self): + def prep(f): # a step whose `to` is not the running kernel any more + cstate(f, to="7.0.14-22-pve") + return "7.0.14-22-pve" + self.k1("to", prep) + + def test_s6_k1_default_before(self): + def prep(f): + cstate(f, default_before="7.0.14-20-pve") + return "7.0.14-20-pve" + self.k1("default_before", prep) + + def test_s6_k1_the_grub_cfg_default_and_the_pinned_file(self): + # the default (both places) names 26; the box runs 28 after a hand boot. The keep computation forgets 26. + def prep(f): + f.tree[osapply.KERNEL_DEFAULT_CFG] = f'GRUB_DEFAULT="gnulinux-advanced-{U}>gnulinux-7.0.14-22-pve-advanced-{U}"\n' + f.cfg = f.render() + return "7.0.14-22-pve" + self.k1("default", prep) + + def test_s6_k1_the_flag(self): + # a flag is a refusal by itself (P2, first); with the keep computation broken, K1 is in the SAME refusal + f = cfake(env={"felhom_next": "7.0.14-20-pve"}) + real = osapply.Kernel.cleanup_keep + osapply.Kernel.cleanup_keep = lambda s_, images, sources: real(s_, images, [(k, w) for k, w in sources if k != "7.0.14-20-pve"]) + try: + rc, rep = run(f) + finally: + osapply.Kernel.cleanup_keep = real + self.assertEqual((rc, rep["refused"]["code"]), (2, "P2"), rep) + self.assertIn("K1 " + img("7.0.14-20-pve") + " is a keep-kernel (the one-shot flag)", rep["refused"]["reason"]) + self.assertFalse([c for c in f.calls if "purge" in c[-1]]) + self.nothing_changed(f) + + def test_s6_k1_an_operators_boot_pin(self): + def prep(f): + f.tree[osapply.KERNEL_PIN_FILE] = "7.0.14-20-pve\n" + return "7.0.14-20-pve" + self.k1("boot pin", prep) + + def test_s6_every_kept_kind_is_in_the_keep_set_and_never_in_the_removal_set(self): + f = cfake() + cstate(f, default_before="7.0.14-20-pve") + f.tree[osapply.KERNEL_PIN_FILE] = "7.0.14-22-pve\n" + f.plan["dry_run"] = True + rc, rep = run(f) + c = rep["kernel_cleanup"] + self.assertEqual(sorted(c["kept"]), sorted(["7.0.14-20-pve", "7.0.14-22-pve", C_FROM, C_RUN])) + self.assertEqual(c["removal"], [img("7.0.2-6-pve")]) + + def test_s6_the_running_kernel_is_not_an_installed_package(self): + f = cfake() + del f.installed[img(C_RUN)] # the box runs a kernel dpkg does not know (an unsigned hand install, say) + f.installed["proxmox-kernel-7.0.14-28-pve-other"] = "7.0.14-28" + self.refused(f, "P3") # P3 is first (the default names a kernel that is not an installed image) … + rc, rep = run(f) + self.assertIn("K1", rep["refused"]["reason"], "… and K1 is in the same refusal") + + def test_no_step_record_keeps_the_next_lower_kernel(self): + f = cstate(cfake(), **{"from": None, "to": None, "default_before": None}) + f.plan["dry_run"] = True + rc, rep = run(f) + self.assertIn("7.0.14-26-pve", rep["kernel_cleanup"]["kept"]) + self.assertIn("next lower", " ".join(rep["kernel_cleanup"]["kept"]["7.0.14-26-pve"])) + + # --- scenario 7: apt would do more (S1, S2) --- + def test_s7_the_simulation_would_remove_something_else(self): + f = cfake(purge_extra_sim=["Purg proxmox-kernel-7.0 [7.0.14-28]", "Remv proxmox-ve [9.2.0]"]) + rep = self.refused(f, "S2") + self.assertIn("proxmox-ve", rep["refused"]["reason"]) + + def test_s7_the_simulation_would_install_something(self): + f = cfake(purge_extra_sim=["Inst proxmox-kernel-7.0.14-28-pve (7.0.14-28 Proxmox Debian Repository:stable [amd64])"]) + rep = self.refused(f, "S1") + self.assertIn("proxmox-kernel-7.0.14-28-pve", rep["refused"]["reason"]) + + def test_s7_the_simulation_fails(self): + self.refused(cfake(purge_sim_rc=100), "S1") + + # --- scenario 8: not our box --- + def test_s8_a_byo_host_is_refused_by_the_appliance_rule(self): + f = cfake() + f.files[osapply.INSTALL_STATE] = json.dumps({"mode": "byo"}) + self.refused(f, "R12") + + def test_s8_a_box_without_the_ring0_mark_does_not_run(self): + f = cfake() + f.files[osapply.TRUST_FILE] = json.dumps({"host_id": "tester-1-d70be4", "ring0_slow_lane": False}) + rep = self.not_run(f, "no-ring0-mark") + self.assertIn("ring 0 only in this release", rep["kernel_cleanup"]["not_run"][0]["reason"]) + g = cfake() # an agent-writable trust file proves nothing + g.stats[osapply.TRUST_FILE] = St(mode=statmod.S_IFREG | 0o666, uid=0) + self.not_run(g, "no-ring0-mark") + h = cfake() # the agent's plan cannot claim the ring + h.files[osapply.TRUST_FILE] = json.dumps({"host_id": "x", "ring0_slow_lane": False}) + h.plan["ring"] = 0 + self.not_run(h, "no-ring0-mark") + + # --- scenario 9: a plan that names packages --- + def test_s9_cleanup_plan_names_refused(self): + for extra in ({"packages": [{"name": img("7.0.14-28-pve"), "version": "7.0.14-28", "origin": "Proxmox Debian Repository"}]}, + {"select": "pending-kernel"}, {"allow_new": ["x"]}, {"signed": {"blob_b64": "eA==", "sig": "x"}}, + {"expect_kver": "7.0.14-20-pve"}, {"dry_run": "yes"}): + f = cfake() + f.plan.update(extra) + self.refused(f, "C1") + + # --- scenario 10: no kernel step ever ran --- + def test_s10_no_state_file_does_not_run(self): + f = cfake() + f.state = None + f.write_state() + rep = self.not_run(f, "no-step") + self.assertIn("no kernel step yet", rep["kernel_cleanup"]["not_run"][0]["reason"]) + + # --- scenario 11: a power cut in the purge --- + def test_s11_a_cut_purge_is_repaired_first_and_the_rule_is_computed_again(self): + f = cfake(purge_rc=100) + rc, rep = run(f) + self.assertEqual(rc, 3, rep) + self.assertEqual(rep["failed"]["step"], "purge") + self.assertEqual(rep["kernel_cleanup"]["outcome"], "failed") + # the cut removed ONE of the three before it stopped + f.installed.pop(self.REMOVE[0]) + f.boot.discard("7.0.14-20-pve") + f.purge_rc, f.purges = 0, [] + f.calls.clear() + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertEqual(rep["repair"]["journal_before"], 1, "repair first: dpkg's journal was replayed") + i_repair = [i for i, c in enumerate(f.calls) if c[-1][-3:-1] == ["--configure", "-a"] or "--configure" in c[-1]][0] + i_purge = [i for i, c in enumerate(f.calls) if "purge" in c[-1]][0] + self.assertLess(i_repair, i_purge, "the repair runs BEFORE the rule is computed and before any purge") + self.assertEqual(f.purges, [self.REMOVE[1:]], "the rule was computed again from the box: the two that are left") + self.assertEqual(rep["kernel_cleanup"]["outcome"], "removed") + + # --- the 20-hour window, the dry run, after-the-purge proof --- + def test_inside_a_steps_20_hour_window_it_does_not_run(self): + f = cfake() + cstate(f, result_at=osapply._iso(f.clock - 3600)) # the step ended good one hour ago + self.not_run(f, "window") + + def test_the_dry_run_reports_and_changes_nothing(self): + f = cfake() + f.plan["dry_run"] = True + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + c = rep["kernel_cleanup"] + self.assertEqual((c["outcome"], c["would"], c["removal"]), ("dry_run", "remove", self.REMOVE)) + self.assertEqual(sorted(c["kept"]), sorted([C_FROM, C_RUN])) + self.nothing_changed(f) + self.assertNotIn("repair", rep, "a dry run does not even repair") + # a dry run of a refusal says which one, with exit 0 + g = cstate(cfake(), phase="judging") + g.plan["dry_run"] = True + rc, rep = run(g) + self.assertEqual((rc, rep["kernel_cleanup"]["would"]), (0, "refuse"), rep) + self.assertEqual([x["code"] for x in rep["kernel_cleanup"]["refusals"]], ["P1"]) + self.nothing_changed(g) + # … and of a box that would not run + h = cfake() + h.files[osapply.TRUST_FILE] = json.dumps({"host_id": "x", "ring0_slow_lane": False}) + h.plan["dry_run"] = True + rc, rep = run(h) + self.assertEqual((rc, rep["kernel_cleanup"]["would"], rep["kernel_cleanup"]["removal"]), (0, "not_run", self.REMOVE), rep) + self.nothing_changed(h) + + def test_a_purge_that_moves_the_default_is_reported_failed_loudly(self): + f = cfake(purge_moves_default=True) # the pinned file vanished in the purge: 10_linux makes the newest the default + rc, rep = run(f) + self.assertEqual(rc, 3, rep) + self.assertEqual(rep["failed"]["step"], "verify") + self.assertIn("pinned default file changed", rep["failed"]["reason"]) + self.assertEqual(f.update_grubs, 0, "never repaired by hand in code") + self.assertTrue(any("FAILED" in l and "verify" in l for l in f.logs)) + + def test_the_apt_lock_is_respected(self): + self.refused(cfake(lock_held=True), "R9") + + def test_both_image_packages_of_one_old_kernel_go_together(self): + f = cfake() + f.installed["proxmox-kernel-7.0.14-20-pve"] = "7.0.14-20" # the unsigned image beside the signed one + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertIn("proxmox-kernel-7.0.14-20-pve", f.purges[0]) + self.assertIn(img("7.0.14-20-pve"), f.purges[0]) + + def test_cleanup_needs_no_new_sudo_line(self): + """The wrapper is reached by `--plan` only: the new mode rides the one sudo line the agent already has.""" + sudo = (HERE / "felhom-agent.sudoers").read_text() + live = [l.strip().rstrip("\\,").strip() for l in sudo.splitlines() if l.strip() and not l.strip().startswith("#")] + self.assertEqual([l for l in live if "felhom-os-apply" in l and "Cmnd_Alias" not in l and "NOPASSWD" not in l], + [osapply.SELF_CHECK_SUDO_LINE], "exactly the one --plan line reaches the wrapper") + self.assertFalse([l for l in live if "kernel-cleanup" in l or + ("apt" in l and ("purge" in l or "remove" in l))], "no sudo line removes a package") + + + +# ---------- the security-profile refresh (R-1388, R-1301, `11` §5.12) ---------- +class PFake(Fake): + """A host with two AppArmor cache folders. `stale`: profile files whose cached copy in the running kernel's folder + is an old compile; the refresh command rewrites the cache from `fresh`. lxc-containers compiles differently every + time (measured on demo-hp). `failed`: unit -> ExecStart program.""" + + def __init__(self, stale=("usr.sbin.chronyd",), failed=None, kernel="7.0.14-28-pve", record=None): + super().__init__() + self.kernel = kernel + self.fresh = {"usr.sbin.chronyd": b"chrony-4.8", "usr.bin.man": b"man", "lxc-containers": b"lxc-0", "nvidia_modprobe": b"nv"} + self.names = {"usr.sbin.chronyd": "/usr/sbin/chronyd", "usr.bin.man": "/usr/bin/man\nman_filter\nman_groff", + "lxc-containers": "lxc-container-default\nlxc-container-default-cgns", "nvidia_modprobe": "nvidia_modprobe\nnvidia_modprobe//kmod"} + self.cache = {"ae1a4050.0": {k: b"old-" + k.encode() for k in self.fresh}, "f6da0254.0": dict(self.fresh)} + for k in stale: + self.cache["f6da0254.0"][k] = b"OLD-COPY-" + k.encode() + self.failed = dict({"chrony.service": "/usr/sbin/chronyd"} if failed is None else failed) + self.restarts, self.parser_runs, self.parser_rc, self.tree = [], 0, 0, {} + if record is not None: + self.tree[osapply.PROFILE_STATE] = json.dumps(record) + self.plan = {"release_id": "profile-refresh", "layer": "host", "mode": "profile-refresh"} + + def isdir(self, p): + return p in (osapply.PROFILE_DIR, osapply.PROFILE_CACHE) or p.startswith(osapply.PROFILE_CACHE + "/") and p.count("/") == 4 + + def list_dir(self, p): + if p == osapply.PROFILE_CACHE: + return sorted(self.cache) + d = p.rsplit("/", 1)[1] + return sorted(list(self.cache.get(d, {})) + [".features"]) + + def read_bytes(self, p): + d, f = p.split("/")[-2:] + return self.cache[d][f] + + def put_file(self, path, data, mode): + self.tree[path] = data.decode() + + def read_file(self, p): + if p in self.tree: + return self.tree[p] + return super().read_file(p) + + def host(self, argv, timeout=600, stdin=None): + if argv[0] == "uname": + self.calls.append(("host", argv)) + return 0, self.kernel + "\n", "" + if argv == osapply.PROFILE_REFRESH_CMD: + self.calls.append(("host", argv)) + self.parser_runs += 1 + if self.parser_rc: + return self.parser_rc, "", "AppArmor parser error for /etc/apparmor.d/x" + self.fresh["lxc-containers"] = b"lxc-%d" % self.parser_runs # never the same binary twice + self.cache["f6da0254.0"] = dict(self.fresh) + return 0, "", "" + if argv[:2] == ["apparmor_parser", "-N"]: + self.calls.append(("host", argv)) + return 0, self.names[argv[2].rsplit("/", 1)[1]] + "\n", "" + if argv[:2] == ["systemctl", "list-units"]: + self.calls.append(("host", argv)) + return 0, "".join(f"{u} loaded failed failed x\n" for u in self.failed), "" + if argv[:2] == ["systemctl", "show"]: + self.calls.append(("host", argv)) + return 0, f"{{ path={self.failed[argv[-1]]} ; argv[]={self.failed[argv[-1]]} -F 1 ; ignore_errors=no }}\n", "" + if argv[:2] == ["systemctl", "restart"]: + self.calls.append(("host", argv)) + self.restarts.append(argv[2]) + return 0, "", "" + return super().host(argv, timeout, stdin) + + +class ProfileRefresh(unittest.TestCase): + """Red-proofs: felhom.eu audits/release-2a-2026-11-14/B/redproof.txt.""" + + def test_a_stale_profile_is_refreshed_and_only_its_failed_unit_restarts(self): + f = PFake(failed={"chrony.service": "/usr/sbin/chronyd", "postfix.service": "/usr/sbin/postfix", "other.service": "/usr/bin/man-not"}) + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + p = rep["profile_refresh"] + self.assertEqual(p["outcome"], "refreshed") + self.assertEqual(f.parser_runs, 1) + self.assertIn(("host", ["apparmor_parser", "--replace", "--skip-read-cache", "--write-cache", "--", "/etc/apparmor.d"]), f.calls) + self.assertEqual(f.cache["f6da0254.0"]["usr.sbin.chronyd"], b"chrony-4.8", "the cache holds the fresh compile") + self.assertEqual(p["rewritten"], ["f6da0254.0/lxc-containers", "f6da0254.0/usr.sbin.chronyd"]) + self.assertEqual(f.restarts, ["chrony.service"], "ONLY the failed unit whose own program's profile was stale") + self.assertEqual([x["unit"] for x in p["restarted"]], ["chrony.service"]) + rec = json.loads(f.tree[osapply.PROFILE_STATE]) + self.assertEqual(rec["kernel"], "7.0.14-28-pve") + + def test_a_fresh_cache_changes_nothing_and_restarts_nothing(self): + f = PFake(stale=()) # chrony is failed for another reason; its profile is NOT stale + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertEqual(rep["profile_refresh"]["rewritten"], ["f6da0254.0/lxc-containers"], "only the file that never compiles the same") + self.assertEqual(f.restarts, [], "a failed unit whose profile was not stale is NOT restarted") + self.assertEqual(f.cache["f6da0254.0"]["usr.sbin.chronyd"], b"chrony-4.8") + + def test_a_stale_profile_of_a_running_program_restarts_nothing(self): + f = PFake(failed={}) # the unit is active: the kernel swaps the profile of a running task itself + rc, rep = run(f) + self.assertEqual((rc, f.restarts, rep["profile_refresh"]["outcome"]), (0, [], "refreshed"), rep) + + def test_a_failed_unit_of_another_program_is_not_restarted(self): + f = PFake(stale=("usr.bin.man",), failed={"chrony.service": "/usr/sbin/chronyd"}) + rc, rep = run(f) + self.assertEqual((rc, f.restarts), (0, []), rep) + g = PFake(stale=("nvidia_modprobe",), failed={"nvidia.service": "nvidia_modprobe"}) # a profile name that is not a path + rc, rep = run(g) + self.assertEqual((rc, g.restarts), (0, []), rep) + + def test_once_per_kernel(self): + f = PFake() + self.assertEqual(run(f)[0], 0) + f.calls.clear() + f.cache["f6da0254.0"]["usr.sbin.chronyd"] = b"OLD-COPY" # even with a stale file: this kernel was done + rc, rep = run(f) + self.assertEqual((rc, rep["profile_refresh"]["outcome"], f.parser_runs), (0, "not_needed", 1), rep) + self.assertFalse([c for c in f.calls if c[1][0] in ("apparmor_parser", "systemctl")], "no parser, no systemctl") + f.kernel = "7.0.14-29-pve" # the box starts on another kernel: again + rc, rep = run(f) + self.assertEqual((rc, rep["profile_refresh"]["outcome"], f.parser_runs), (0, "refreshed", 2), rep) + self.assertEqual(json.loads(f.tree[osapply.PROFILE_STATE])["kernel"], "7.0.14-29-pve") + + def test_a_failed_parser_writes_no_record_so_the_next_start_tries_again(self): + f = PFake() + f.parser_rc = 1 + rc, rep = run(f) + self.assertEqual((rc, rep["failed"]["step"], rep["profile_refresh"]["outcome"]), (3, "profile-refresh", "failed"), rep) + self.assertNotIn(osapply.PROFILE_STATE, f.tree) + self.assertEqual(f.restarts, []) + f.parser_rc = 0 + self.assertEqual(run(f)[1]["profile_refresh"]["outcome"], "refreshed") + + def test_the_plan_names_nothing(self): + for extra in ({"profiles": ["usr.sbin.chronyd"]}, {"units": ["ssh.service"]}, {"packages": [{"name": "chrony", "version": "4.8", "origin": "Debian"}]}, + {"force": True}, {"dry_run": 1}, {"layer": "guest"}): + f = PFake() + f.plan.update(extra) + rc, rep = run(f) + self.assertEqual(rc, 2, rep) + self.assertIn(rep["refused"]["code"], ("F1", "R11"), rep) + self.assertEqual((f.parser_runs, f.restarts), (0, [])) + + def test_a_byo_host_is_refused(self): + f = PFake() + f.files[osapply.INSTALL_STATE] = json.dumps({"mode": "byo"}) + rc, rep = run(f) + self.assertEqual((rc, rep["refused"]["code"], f.parser_runs), (2, "R12", 0), rep) + + def test_the_dry_run_changes_nothing(self): + f = PFake() + f.plan["dry_run"] = True + rc, rep = run(f) + self.assertEqual((rc, rep["profile_refresh"]["outcome"], rep["profile_refresh"]["would"]), (0, "dry_run", "refresh"), rep) + self.assertEqual((f.parser_runs, f.restarts), (0, [])) + self.assertNotIn(osapply.PROFILE_STATE, f.tree) + + def test_no_more_than_three_restarts(self): + f = PFake(failed={f"chrony{i}.service": "/usr/sbin/chronyd" for i in range(5)}) + rc, rep = run(f) + self.assertEqual((rc, len(f.restarts), len(rep["profile_refresh"]["not_restarted"])), (0, 3, 2), rep) + + def test_profile_refresh_changes_no_package(self): + f = PFake() + run(f) + self.assertFalse([c for c in f.calls if c[1][0] in ("apt-get", "dpkg", "apt")], "no package command at all") + if __name__ == "__main__": unittest.main() diff --git a/go.mod b/go.mod index 238101c..0418c52 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitea.dooplex.hu/admin/felhom-agent -go 1.27.3 +go 1.27.4 require ( golang.org/x/crypto v0.58.0 diff --git a/internal/osupdate/kernel.go b/internal/osupdate/kernel.go index 4219b37..af8f354 100644 --- a/internal/osupdate/kernel.go +++ b/internal/osupdate/kernel.go @@ -21,6 +21,8 @@ import ( "fmt" "log/slog" "regexp" + "sort" + "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" @@ -444,3 +446,129 @@ func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParam } return l.finish(ctx, lg, rep) } + +// ---- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ---- +// +// The rule, its refusals and "may this box do it" live in the root wrapper (mode kernel-cleanup); the plan names NO +// package. This file decides only WHEN: in the night leg, after a healthy host step, never in a night with a kernel +// step. The wrapper adds its own 20-hour window after a step. + +// KernelCleanupResult is the wrapper's kernel_cleanup object. +type KernelCleanupResult struct { + Outcome string `json:"outcome"` // removed | nothing | not_run | dry_run | failed + DryRun bool `json:"dry_run"` + Would string `json:"would"` // dry run only: remove | nothing | refuse | not_run + NotRun []struct { + Code string `json:"code"` + Reason string `json:"reason"` + } `json:"not_run"` + Refusals []struct { + Code string `json:"code"` + Reason string `json:"reason"` + } `json:"refusals"` + Kept map[string][]string `json:"kept"` + Removal []string `json:"removal"` + Removed []string `json:"removed"` + BytesGained *int64 `json:"bytes_gained"` + Leftovers []string `json:"leftovers"` +} + +// kernelCleanupSkip says why tonight's pass must not call the clean-up ("" = call it). +func kernelCleanupSkip(appliance bool, trigger string, host, pve, kernel Report, blk hub.WireOSUpdate, okStep func(Report) bool) string { + switch { + case trigger != "night": + return "it runs only in the night leg (never a debug pass)" + case !appliance: + return "not an appliance install (a BYO host belongs to its owner)" + case !blk.Enabled: + return "OS updates are switched off for this box" + case host.Layer == "" || !okStep(host): + return "no healthy host step this pass" + case pve.Layer != "" && !okStep(pve): + return "the Proxmox step did not end healthy" + case kernel.Layer != "": + return "a kernel step ran in this night (" + kernel.Outcome + ") — never in the same night" + } + if due, _ := kernelDue(blk, trigger); due { + return "a kernel step is due tonight — never in the same night" + } + return "" +} + +// runKernelCleanup calls the wrapper once. The hub hears about it only when there is something to tell: kernels were +// removed (outcome "cleaned"), the wrapper refused, or the purge failed. "Nothing to remove" and "not run" are log +// lines — one report per box per night would bury the kernel step's own line on the hub's System page. +func (l *Leg) runKernelCleanup(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report { + lg := l.log().With("run", runID, "layer", LayerKernel, "mode", "kernel-cleanup", "vmid", vmid, "trigger", trigger) + start := l.now() + wr, err := l.call(ctx, runID, kernelPlan("kernel-cleanup", vmid, map[string]any{"release_id": "kernel-cleanup", + "run_id": runID, "trigger": trigger, "ring": blk.Ring})) + // NO kernel view and no kernel version in this report: the hub reads a kernel-layer report's `kernel.to` / + // `release_id` as "the kernel this STEP was for" (kernelOf), and a clean-up is not a step for any kernel. + rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "kernel-cleanup", + ReleaseID: "kernel-cleanup", PassSeconds: wr.PassSeconds} + var res KernelCleanupResult + _ = json.Unmarshal(wr.KernelCleanup, &res) + took := l.now().Sub(start).Round(time.Second).String() + switch { + case err != nil: + lg.Warn("osupdate: kernel clean-up — the wrapper gave no answer; not retried tonight", "err", err, "took", took) + rep.Outcome, rep.HealthReason = "failed", "kernel clean-up: "+err.Error() + return l.finish(ctx, lg, rep) + case wr.refused(): + lg.Warn("osupdate: kernel clean-up REFUSED — nothing was removed; not retried tonight", "refused", string(wr.Refused), "took", took) + rep.Outcome, rep.Refused, rep.HealthReason = "refused", wr.Refused, "the clean-up of old kernels was refused; nothing was removed" + return l.finish(ctx, lg, rep) + case wr.failed(): + lg.Error("osupdate: kernel clean-up FAILED — look at the box; not retried tonight", "failed", string(wr.Failed), "took", took) + rep.Outcome, rep.Refused, rep.HealthReason = "failed", wr.Failed, "the clean-up of old kernels failed" + return l.finish(ctx, lg, rep) + } + switch res.Outcome { + case "removed": + gained := "unknown" + if res.BytesGained != nil { + gained = fmt.Sprintf("%.1f GiB", float64(*res.BytesGained)/(1<<30)) + } + rep.Outcome, rep.Healthy = "cleaned", true + rep.HealthReason = truncate(fmt.Sprintf("removed %d old kernel package(s): %s; kept %s; %s gained; %d left-over path(s)", + len(res.Removed), strings.Join(res.Removed, ", "), strings.Join(sortedKeys(res.Kept), ", "), gained, len(res.Leftovers)), 600) + lg.Info("osupdate: kernel clean-up — old kernels removed", "removed", len(res.Removed), "names", strings.Join(res.Removed, ","), + "kept", strings.Join(sortedKeys(res.Kept), ","), "bytes_gained", res.BytesGained, "leftovers", len(res.Leftovers), "took", took) + return l.finish(ctx, lg, rep) + case "nothing": + lg.Info("osupdate: kernel clean-up — nothing to remove", "kept", strings.Join(sortedKeys(res.Kept), ","), "took", took) + case "not_run": + code, why := "", "" + if len(res.NotRun) > 0 { + code, why = res.NotRun[0].Code, res.NotRun[0].Reason + } + lg.Info("osupdate: kernel clean-up — not run on this box tonight", "code", code, "reason", why, "took", took) + default: + lg.Warn("osupdate: kernel clean-up — the wrapper answered with no outcome this agent knows (an older wrapper?)", "outcome", res.Outcome, "took", took) + } + return Report{} +} + +// KernelCleanupDryRun asks the wrapper what the clean-up WOULD do, and changes nothing (the debug tool: `--selftest +// kernel-cleanup-dry-run`). The raw kernel_cleanup object, or the refusal. +func (l *Leg) KernelCleanupDryRun(ctx context.Context, vmid int) (json.RawMessage, error) { + wr, err := l.call(ctx, "kcdry"+l.now().UTC().Format("150405"), kernelPlan("kernel-cleanup", vmid, + map[string]any{"release_id": "kernel-cleanup", "dry_run": true})) + if err != nil { + return nil, err + } + if wr.refused() { + return nil, fmt.Errorf("kernel-cleanup dry run refused: %s", wr.Refused) + } + return wr.KernelCleanup, nil +} + +func sortedKeys(m map[string][]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/osupdate/kernel_test.go b/internal/osupdate/kernel_test.go index f9ca9f8..d5d8c0a 100644 --- a/internal/osupdate/kernel_test.go +++ b/internal/osupdate/kernel_test.go @@ -28,13 +28,23 @@ func tonight(ring int) *hub.WireOSUpdate { func kernelCalls(w *fakeWrapper) []string { var m []string for _, p := range w.plans { - if p["layer"] == LayerKernel { + if p["layer"] == LayerKernel && p["mode"] != "kernel-cleanup" { // the kernel STEP's calls (the clean-up: cleanupCalls) m = append(m, p["mode"].(string)) } } return m } +func cleanupCalls(w *fakeWrapper) []map[string]any { + var m []map[string]any + for _, p := range w.plans { + if p["layer"] == LayerKernel && p["mode"] == "kernel-cleanup" { + m = append(m, p) + } + } + return m +} + // Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the // kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night. func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) { @@ -326,3 +336,177 @@ func TestKernelSet(t *testing.T) { t.Fatal("a non-kernel string must give no set") } } + +// ---- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ---- + +func cleanupRep(js string) map[string][]WrapperReport { + return map[string][]WrapperReport{"kernel-cleanup": {{KernelCleanup: json.RawMessage(js)}}} +} + +// The night: the clean-up is the LAST call, after the host step, exactly once, and its plan names NO package. +// COMPANION RED-PROOF (observed): make kernelCleanupSkip return "" before its checks → the BYO / debug / unhealthy +// sub-cases of TestKernelCleanup_WhenItMayNotRun fail; put the call before runFast → this test's order check fails. +func TestKernelCleanup_NightOrderOnceAndNoNames(t *testing.T) { + w := &fakeWrapper{t: t, kernelRep: cleanupRep(`{"outcome":"removed","removed":["proxmox-kernel-7.0.14-20-pve-signed"],` + + `"kept":{"7.0.14-22-pve":["running"],"7.0.2-6-pve":["from"]},"bytes_gained":1073741824,"leftovers":[]}`)} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) + p := l.Run(context.Background(), 9201, "night") + cc := cleanupCalls(w) + if len(cc) != 1 { + t.Fatalf("clean-up calls = %d, want exactly one a night", len(cc)) + } + if last := w.plans[len(w.plans)-1]; last["mode"] != "kernel-cleanup" { + t.Fatalf("the clean-up must be the night's last call (after the host step), last = %v", last) + } + hostAt, cleanAt := -1, -1 + for i, x := range w.plans { + if x["layer"] == LayerHost { + hostAt = i + } + if x["mode"] == "kernel-cleanup" { + cleanAt = i + } + } + if hostAt < 0 || cleanAt < hostAt { + t.Fatalf("the clean-up ran before the host step: host %d clean-up %d", hostAt, cleanAt) + } + plan := cc[0] + if pk, _ := json.Marshal(plan["packages"]); string(pk) != "[]" { + t.Fatalf("the clean-up plan names packages: %s", pk) + } + for _, k := range []string{"select", "signed", "allow_new", "expect_kver", "dry_run"} { + if _, ok := plan[k]; ok { + t.Fatalf("the clean-up plan carries %q: %v", k, plan) + } + } + if plan["lane"] != "slow" || plan["layer"] != LayerKernel { + t.Fatalf("plan = %v", plan) + } + // the hub hears "cleaned" with what was removed, kept and gained — and NO kernel view / version (the hub would read + // it as a kernel step for that kernel) + rep := p.KernelCleanup + if rep.Outcome != "cleaned" || !rep.Healthy || rep.Mode != "kernel-cleanup" || rep.Layer != LayerKernel { + t.Fatalf("report = %+v", rep) + } + for _, want := range []string{"removed 1 old kernel package(s): proxmox-kernel-7.0.14-20-pve-signed", "kept 7.0.14-22-pve, 7.0.2-6-pve", "1.0 GiB gained"} { + if !strings.Contains(rep.HealthReason, want) { + t.Fatalf("report reason %q lacks %q", rep.HealthReason, want) + } + } + lastRep := h.reports[len(h.reports)-1] + if lastRep.Mode != "kernel-cleanup" || lastRep.Outcome != "cleaned" { + t.Fatalf("the hub did not hear the clean-up: %+v", lastRep) + } + if len(lastRep.Kernel) != 0 || kverRE.MatchString(lastRep.ReleaseID) { + t.Fatalf("the clean-up's report names a kernel (the hub reads that as a kernel step): %+v", lastRep) + } + // the same night again: the pass is skipped as a whole — never a second clean-up + l.Run(context.Background(), 9201, "night") + if len(cleanupCalls(w)) != 1 { + t.Fatalf("a second clean-up in the same night: %d calls", len(cleanupCalls(w))) + } +} + +// "Nothing to remove" and "not run" are log lines, never a report (one a night would bury the kernel step's own line). +func TestKernelCleanup_NothingAndNotRunSendNoReport(t *testing.T) { + for name, js := range map[string]string{ + "nothing": `{"outcome":"nothing","removal":[],"kept":{"7.0.14-22-pve":["running"]}}`, + "not_run": `{"outcome":"not_run","not_run":[{"code":"no-ring0-mark","reason":"ring 0 only in this release"}]}`, + } { + w := &fakeWrapper{t: t, kernelRep: cleanupRep(js)} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) + p := l.Run(context.Background(), 9201, "night") + if len(cleanupCalls(w)) != 1 { + t.Fatalf("%s: the wrapper was not asked", name) + } + if p.KernelCleanup.Layer != "" { + t.Fatalf("%s: a report was made: %+v", name, p.KernelCleanup) + } + for _, r := range h.reports { + if r.Mode == "kernel-cleanup" { + t.Fatalf("%s: the hub got a clean-up report: %+v", name, r) + } + } + } +} + +// A refusal and a failure reach the hub (the operator), once, and are not retried. +func TestKernelCleanup_RefusedAndFailedReachTheHub(t *testing.T) { + for name, wr := range map[string]WrapperReport{ + "refused": {Refused: json.RawMessage(`{"code":"P3","reason":"the pinned default and grub.cfg disagree"}`)}, + "failed": {Failed: json.RawMessage(`{"rc":3,"step":"verify","reason":"the default in grub.cfg moved"}`), KernelCleanup: json.RawMessage(`{"outcome":"failed"}`)}, + } { + w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-cleanup": {wr}}} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) + p := l.Run(context.Background(), 9201, "night") + if p.KernelCleanup.Outcome != name || p.KernelCleanup.Healthy || len(p.KernelCleanup.Refused) == 0 { + t.Fatalf("%s: report = %+v", name, p.KernelCleanup) + } + if last := h.reports[len(h.reports)-1]; last.Mode != "kernel-cleanup" || last.Outcome != name { + t.Fatalf("%s: the hub did not hear it: %+v", name, last) + } + if len(cleanupCalls(w)) != 1 { + t.Fatalf("%s: retried the same night (%d calls)", name, len(cleanupCalls(w))) + } + } +} + +// When it may NOT run: never with a kernel step in the night (due, staged, or reported), not in a debug pass, not on a +// BYO host, not with OS updates off, not after an unhealthy host step. +func TestKernelCleanup_WhenItMayNotRun(t *testing.T) { + type tc struct { + blk *hub.WireOSUpdate + trigger string + prep func(*fakeWrapper, *Leg) + } + bad := &Health{DockerOK: false} + cases := map[string]tc{ + "a kernel step is due tonight (ring 0)": {tonight(0), "night", func(w *fakeWrapper, _ *Leg) { + w.kernelRep = map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}, + "apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}}, + "kernel-reboot": {{Kernel: kview("oneshot")}}} + }}, + "a kernel step is due tonight (ring 1, nothing staged)": {tonight(1), "night", func(w *fakeWrapper, _ *Leg) { + w.kernelRep = map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}} + }}, + "a debug pass": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "debug", nil}, + "a BYO host": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night", func(_ *fakeWrapper, l *Leg) { l.Appliance = false }}, + "updates off": {&hub.WireOSUpdate{Ring: 0, Enabled: false}, "night", nil}, + "host unhealthy": {&hub.WireOSUpdate{Ring: 0, Enabled: true, HostRelease: &hub.WireOSRelease{ID: "os-host-1", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "u3", Origin: "Debian"}}}}, "night", func(w *fakeWrapper, _ *Leg) { + w.applyRep[LayerHost] = WrapperReport{Upgraded: []Package{{Name: "openssl", Version: "u3"}}, HealthAfter: &Health{HostServices: map[string]string{"pveproxy": "failed"}, GuestRunning: yes(), Guest: bad}} + w.healthSeq[LayerHost] = []*Health{{HostServices: map[string]string{"pveproxy": "failed"}, GuestRunning: yes(), Guest: bad}} + }}, + } + for name, c := range cases { + w := &fakeWrapper{t: t} + l, _ := newLeg(t, w, c.blk) + if c.prep != nil { + c.prep(w, l) + } + l.HealthWait = time.Second + l.Run(context.Background(), 9201, c.trigger) + if n := len(cleanupCalls(w)); n != 0 { + t.Fatalf("%s: the clean-up ran (%d call(s)); calls = %s", name, n, calls(w)) + } + } +} + +// The dry run is the same mode with dry_run: true — and it sends nothing to the hub. +func TestKernelCleanup_DryRun(t *testing.T) { + w := &fakeWrapper{t: t, kernelRep: cleanupRep(`{"outcome":"dry_run","would":"remove","removal":["proxmox-kernel-7.0.14-20-pve-signed"]}`)} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) + raw, err := l.KernelCleanupDryRun(context.Background(), 9201) + if err != nil || !strings.Contains(string(raw), `"would":"remove"`) { + t.Fatalf("dry run = %s, %v", raw, err) + } + cc := cleanupCalls(w) + if len(cc) != 1 || cc[0]["dry_run"] != true { + t.Fatalf("the dry run's plan = %v", cc) + } + if pk, _ := json.Marshal(cc[0]["packages"]); string(pk) != "[]" { + t.Fatalf("the dry run's plan names packages: %s", pk) + } + if len(h.reports) != 0 { + t.Fatalf("a dry run reported to the hub: %+v", h.reports) + } +} diff --git a/internal/osupdate/leg.go b/internal/osupdate/leg.go index 92496db..1ac2926 100644 --- a/internal/osupdate/leg.go +++ b/internal/osupdate/leg.go @@ -134,7 +134,12 @@ type WrapperReport struct { // found after a boot; OutcomeHint "nothing" when no kernel was pending. Kernel json.RawMessage `json:"kernel"` KernelEvent string `json:"kernel_event"` - OutcomeHint string `json:"outcome_hint"` + // KernelCleanup (R-1368, `11` §5.11.1): mode kernel-cleanup's result {outcome, dry_run, kept, removal, removed, + // bytes_gained, not_run, refusals, leftovers}. + KernelCleanup json.RawMessage `json:"kernel_cleanup"` + // ProfileRefresh (R-1388, `11` §5.12): mode profile-refresh's result {outcome, kernel, rewritten, restarted, seconds}. + ProfileRefresh json.RawMessage `json:"profile_refresh"` + OutcomeHint string `json:"outcome_hint"` // R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the // agent process that started the pass. ReleaseID / VMID were always in the report. RunID string `json:"run_id"` @@ -531,6 +536,8 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap // Pass is one leg's reports; an empty Layer means the step did not run. type Pass struct { Guest, Host, Docker, PVE, Kernel Report + // KernelCleanup (R-1368): the clean-up of old kernels. Layer "" when it did not run or had nothing to tell the hub. + KernelCleanup Report } // Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0 @@ -582,6 +589,15 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass { default: p.Kernel = l.runKernel(ctx, g.RunID, vmid, trigger, blk) } + // R-1368 (`09` §3 decision 286, `11` §5.11.1): the clean-up of old kernels — its own step, at most once a night (this + // pass is the night's only one), after the host step ended healthy, NEVER in a night with a kernel step. Whether + // this box may do it at all is the root wrapper's answer (its root-owned ring-0 mark), never the agent's. + // Pinned by TestKernelCleanup_*. + if why := kernelCleanupSkip(l.Appliance, trigger, h, p.PVE, p.Kernel, blk, okStep); why != "" { + lg.Info("osupdate: kernel clean-up skipped — " + why) + } else { + p.KernelCleanup = l.runKernelCleanup(ctx, g.RunID, vmid, trigger, blk) + } return p } diff --git a/internal/osupdate/leg_test.go b/internal/osupdate/leg_test.go index d4b1490..aed090e 100644 --- a/internal/osupdate/leg_test.go +++ b/internal/osupdate/leg_test.go @@ -26,6 +26,7 @@ type fakeWrapper struct { keep bool // R-868: like the real wrapper, keep an apply report beside the plan pveGateHeld bool kernelRep map[string][]WrapperReport // R-836: per kernel-layer mode, successive answers (the last one repeats) + profileRep *WrapperReport // R-1388: the answer to mode profile-refresh } func yes() *bool { b := true; return &b } @@ -71,6 +72,10 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt } var rep WrapperReport switch plan["mode"] { + case "profile-refresh": + if f.profileRep != nil { + rep = *f.profileRep + } case "inventory": rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthBefore: ok, HealthAfter: ok, Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}} @@ -176,8 +181,9 @@ func TestRing0_OneCallPerLayer(t *testing.T) { if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy { t.Fatalf("guest %+v\nhost %+v", g, ho) } - if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply,pve:apply" { - t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore, the ring-0 docker step and the pve step", calls(w)) + // R-1368: the clean-up of old kernels is the night's last call when no kernel step is due + if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply,pve:apply,kernel:kernel-cleanup" { + t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore, the ring-0 docker step, the pve step, the kernel clean-up", calls(w)) } for _, p := range w.plans[:2] { if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 { @@ -221,7 +227,8 @@ func TestRing1_NoReleaseIsInventory(t *testing.T) { w := &fakeWrapper{t: t, pending: pend} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) g, ho := run2(l, "night") - if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" { + // R-1368: a ring-1 box asks the wrapper too — whether it may clean up is the wrapper's answer (its root-owned mark) + if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory,kernel:kernel-cleanup" { t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w)) } } diff --git a/internal/osupdate/profilerefresh.go b/internal/osupdate/profilerefresh.go new file mode 100644 index 0000000..814d219 --- /dev/null +++ b/internal/osupdate/profilerefresh.go @@ -0,0 +1,101 @@ +package osupdate + +// The security-profile refresh (R-1388, R-1301, `11` §5.12). After a kernel went back and forth a box can load an OLD +// compiled AppArmor profile for a program (on the N100 this stopped the time service). The root wrapper's mode +// profile-refresh loads every profile from a fresh compile, once per kernel the box starts on, and restarts only a +// failed service whose own program's profile was stale. This file decides WHEN to ask: at every agent start and after +// a config bundle was installed (a new wrapper may know the mode the old one refused) — the wrapper itself answers +// "not needed" when this kernel was already done. Nothing is sent to the hub (no hub field exists for it; R-1383). + +import ( + "context" + "encoding/json" + "strings" + "time" +) + +// ProfileRefreshResult is the wrapper's profile_refresh object. +type ProfileRefreshResult struct { + Outcome string `json:"outcome"` // refreshed | not_needed | dry_run | failed + Reason string `json:"reason"` + Kernel string `json:"kernel"` + Rewritten []string `json:"rewritten"` + Restarted []struct { + Unit string `json:"unit"` + Program string `json:"program"` + RC int `json:"rc"` + Active bool `json:"active"` + } `json:"restarted"` + Seconds float64 `json:"seconds"` +} + +// ProfileGate takes the heavy-op gate (a backup, a restore-test or an OS step must not run beside the refresh). +type ProfileGate func() (release func(), busy string, ok bool) + +// ProfileRefresh asks the wrapper once. It returns the outcome ("" when the wrapper could not be asked or refused). +// It waits for the gate: up to `tries` attempts, `wait` apart. why is the log's short reason for this call. +func (l *Leg) ProfileRefresh(ctx context.Context, why string, gate ProfileGate, tries int, wait time.Duration) string { + lg := l.log().With("mode", "profile-refresh", "why", why) + if !l.Appliance { + lg.Info("osupdate: profile refresh skipped — not an appliance install (a BYO host belongs to its owner)") + return "" + } + var release func() + for i := 1; ; i++ { + if gate == nil { + break + } + r, busy, ok := gate() + if ok { + release = r + break + } + if i >= tries || ctx.Err() != nil { + lg.Warn("osupdate: profile refresh NOT run — the box stayed busy; the next agent start asks again", "busy", busy, "tries", i) + return "" + } + lg.Info("osupdate: profile refresh waits — a heavy operation runs", "busy", busy, "try", i) + l.sleep(ctx, wait) + } + if release != nil { + defer release() + } + unlock := l.lockPass(true) // never beside an OS-update pass + defer unlock() + start := l.now() + wr, err := l.call(ctx, "prefresh"+l.now().UTC().Format("150405"), map[string]any{"release_id": "profile-refresh", + "layer": LayerHost, "mode": "profile-refresh"}) + took := l.now().Sub(start).Round(100 * time.Millisecond).String() + switch { + case err != nil: + lg.Warn("osupdate: profile refresh — the wrapper gave no answer; the next agent start asks again", "err", err, "took", took) + return "" + case wr.refused(): + lg.Info("osupdate: profile refresh refused (an older wrapper that does not know the mode, or a BYO host)", "refused", string(wr.Refused)) + return "" + case wr.failed(): + lg.Error("osupdate: profile refresh FAILED — the profiles are as before; the next agent start asks again", "failed", string(wr.Failed), "took", took) + return "failed" + } + var res ProfileRefreshResult + _ = json.Unmarshal(wr.ProfileRefresh, &res) + switch res.Outcome { + case "refreshed": + units := make([]string, 0, len(res.Restarted)) + for _, u := range res.Restarted { + units = append(units, u.Unit) + } + if len(units) > 0 { + lg.Warn("osupdate: profile refresh — security profiles reloaded from a fresh compile; a FAILED service whose profile was stale was restarted", + "kernel", res.Kernel, "restarted", strings.Join(units, ","), "rewritten", len(res.Rewritten), "wrapper_seconds", res.Seconds, "took", took) + } else { + lg.Info("osupdate: profile refresh — security profiles reloaded from a fresh compile", "kernel", res.Kernel, + "rewritten", len(res.Rewritten), "wrapper_seconds", res.Seconds, "took", took) + } + case "not_needed": + lg.Info("osupdate: profile refresh not needed", "kernel", res.Kernel, "reason", res.Reason) + default: + lg.Warn("osupdate: profile refresh — the wrapper answered with no outcome this agent knows", "outcome", res.Outcome) + } + return res.Outcome +} diff --git a/internal/osupdate/profilerefresh_test.go b/internal/osupdate/profilerefresh_test.go new file mode 100644 index 0000000..1713462 --- /dev/null +++ b/internal/osupdate/profilerefresh_test.go @@ -0,0 +1,97 @@ +package osupdate + +import ( + "context" + "encoding/json" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// ---- the security-profile refresh (R-1388, R-1301, `11` §5.12) ---- + +func profileCalls(w *fakeWrapper) []map[string]any { + var m []map[string]any + for _, p := range w.plans { + if p["mode"] == "profile-refresh" { + m = append(m, p) + } + } + return m +} + +func refreshed() *WrapperReport { + return &WrapperReport{ProfileRefresh: json.RawMessage(`{"outcome":"refreshed","kernel":"7.0.14-28-pve","rewritten":["f6da0254.0/usr.sbin.chronyd"],` + + `"restarted":[{"unit":"chrony.service","program":"/usr/sbin/chronyd","rc":0,"active":true}],"seconds":0.5}`)} +} + +// One call, a host-layer plan that names NOTHING, and nothing to the hub. +// COMPANION RED-PROOF (observed): add "units" to the plan → this test fails; drop the Appliance check → the BYO test fails. +func TestProfileRefresh_OneCallNoNamesNoHubReport(t *testing.T) { + w := &fakeWrapper{t: t, profileRep: refreshed()} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) + if got := l.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "refreshed" { + t.Fatalf("outcome = %q", got) + } + pc := profileCalls(w) + if len(pc) != 1 { + t.Fatalf("calls = %d", len(pc)) + } + if len(pc[0]) != 3 || pc[0]["layer"] != LayerHost || pc[0]["release_id"] != "profile-refresh" { + t.Fatalf("the plan carries more than release_id, layer and mode: %v", pc[0]) + } + if len(h.reports) != 0 { + t.Fatalf("the hub got a report: %+v", h.reports) + } +} + +func TestProfileRefresh_ByoHostIsNeverAsked(t *testing.T) { + w := &fakeWrapper{t: t, profileRep: refreshed()} + l, _ := newLeg(t, w, nil) + l.Appliance = false + if got := l.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "" || len(profileCalls(w)) != 0 { + t.Fatalf("a BYO host was asked: %q %v", got, profileCalls(w)) + } +} + +// The heavy-op gate: busy → wait and ask again; still busy after the tries → not run (the next start asks again). +// COMPANION RED-PROOF (observed): ignore `ok` from the gate → the "always busy" half fails. +func TestProfileRefresh_WaitsForTheHeavyOpGate(t *testing.T) { + w := &fakeWrapper{t: t, profileRep: refreshed()} + l, _ := newLeg(t, w, nil) + n, released := 0, 0 + gate := func() (func(), string, bool) { + n++ + if n < 3 { + return nil, "primary-backup", false + } + return func() { released++ }, "", true + } + if got := l.ProfileRefresh(context.Background(), "agent start", gate, 5, time.Minute); got != "refreshed" { + t.Fatalf("outcome = %q after %d gate tries", got, n) + } + if n != 3 || released != 1 || len(profileCalls(w)) != 1 { + t.Fatalf("gate tries %d, released %d, calls %d", n, released, len(profileCalls(w))) + } + w2 := &fakeWrapper{t: t, profileRep: refreshed()} + l2, _ := newLeg(t, w2, nil) + busy := func() (func(), string, bool) { return nil, "restore-test", false } + if got := l2.ProfileRefresh(context.Background(), "agent start", busy, 3, time.Minute); got != "" || len(profileCalls(w2)) != 0 { + t.Fatalf("ran beside a heavy operation: %q %d call(s)", got, len(profileCalls(w2))) + } +} + +// An older wrapper refuses the mode: logged, nothing else. A failure is said as "failed". +func TestProfileRefresh_RefusedAndFailed(t *testing.T) { + w := &fakeWrapper{t: t, profileRep: &WrapperReport{Refused: json.RawMessage(`{"code":"R11","reason":"unknown mode 'profile-refresh'"}`)}} + l, _ := newLeg(t, w, nil) + if got := l.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "" { + t.Fatalf("refused → %q", got) + } + w2 := &fakeWrapper{t: t, profileRep: &WrapperReport{Failed: json.RawMessage(`{"rc":1,"step":"profile-refresh"}`)}} + l2, _ := newLeg(t, w2, nil) + if got := l2.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "failed" { + t.Fatalf("failed → %q", got) + } +}