From acccb66bd33716f9cf06dbccbf2da3ebd2f195c6 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 6 Oct 2026 19:15:06 +0200 Subject: [PATCH] R-528 (09 decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill felhom-os-apply: a docker-layer apply runs oom_check() after health_after and reports "oom_check": {result pass|fail|error, oom_killed, oom_event, exit_code, image, detail}. One throwaway container (the controller's image, --pull never, --network none, 64m cap, label felhom.oomcheck=1) runs dd bs=200M; pass only with OOMKilled=true AND the oom event (read after a 2 s settle, --until = guest epoch + 1: measured on demo-hp, an --until taken right after the run missed the event). docker rm -f always runs in a finally; every call is bounded (<= 90 s). It never changes the step's outcome or health. New wrapper-only mode "oom-check" (docker layer) runs the check alone; check_guest etc. still apply. Agent: WrapperReport/Report gain OOMCheck (json:"oom_check"), copied unchanged in runLayer and in the kept-copy path. Tests: 9 wrapper tests + 2 Go tests, each red-proofed (audits/readback-2026-10-07/F/red-*.txt). Also: test_felhom_os_apply.py's `if __name__` sat mid-file, so 11 tests (UnsentReport, SaveReportOnDisk, AgentDiesMidPass, CrashLeftTheJournal) never ran as a script or from TestWrapperSuite; moved to the end (they pass). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- configs/felhom-os-apply | 105 ++++++++++++++++++++- configs/test_felhom_os_apply.py | 153 ++++++++++++++++++++++++++++++- internal/osupdate/leg.go | 15 +++ internal/osupdate/leg_test.go | 45 ++++++++- internal/osupdate/unsent.go | 1 + internal/osupdate/unsent_test.go | 21 +++++ 6 files changed, 335 insertions(+), 5 deletions(-) diff --git a/configs/felhom-os-apply b/configs/felhom-os-apply index 4606ff6..3ac7397 100755 --- a/configs/felhom-os-apply +++ b/configs/felhom-os-apply @@ -32,6 +32,14 @@ # held packages, kernel taint, the crash guard; guest Debian, Docker engine, containerd, live-restore. # live-restore-on (v0.142.0, layer guest) the ONE-TIME step of `09` decision 87: merge `"live-restore": true` # into the guest's /etc/docker/daemon.json and `systemctl reload docker`. NEVER a restart (R-835). +# oom-check (R-528, `09` decision 157; layer docker, lane slow) ONLY the memory-kill check below: no apt, no engine +# change, no authority needed. Wrapper-only — the agent never writes this plan; run it by hand as root. +# R-528 (`09` decision 157): a docker-layer APPLY also runs `oom_check()` after health_after and reports it as +# "oom_check": {"result": "pass"|"fail"|"error", "oom_killed": bool, "oom_event": bool, "exit_code": int|null, +# "image": str|null, "detail": str} +# — one throwaway container (the controller's own image, no network / volume / port, 64 MB cap) is made to exceed its +# memory; "pass" only when the engine says OOMKilled=true AND emits the `oom` event. It never changes the step's +# outcome or health: the hub decides whether the engine set can be approved. Pinned by the OOMCheck tests. # Output: log lines on stderr and the journal (tag felhom-os-apply); the LAST stdout line is # OSAPPLY-REPORT # which is what the agent parses. Exit 0 = done; 2 = refused (nothing changed); 3 = failed during install. @@ -68,6 +76,16 @@ JOURNAL_MARK = "@@FELHOM-DPKG-JOURNAL@@" DPKG_STATE_SCRIPT = "dpkg --audit; echo " + JOURNAL_MARK + "; ls -A /var/lib/dpkg/updates 2>/dev/null; true" # The installer's ROOT-OWNED record (felhom-host-install.sh `state_set mode`); the agent cannot write it. INSTALL_STATE = "/var/lib/felhom-install/state.json" +# R-528: the memory-kill check (oom_check). One 200 MB block under a 64 MB cap: measured on Docker 29.8.2 to be +# OOM-killed with OOMKilled=true and an `oom` event. Every call is bounded: timeouts (the clock read twice) + the +# settle wait stay within 90 s. +OOMCHECK_PREFIX = "felhom-oomcheck-" +OOMCHECK_SCRIPT = "dd if=/dev/zero of=/dev/null bs=200M count=1" +OOMCHECK_TIMEOUTS = {"image": 10, "clock": 5, "run": 30, "inspect": 10, "events": 10, "rm": 15} +# Measured on demo-hp 9201 (Docker 29.8.2, 2026-10-06, audits/readback-2026-10-07/F/F1, F2): an `--until` taken right +# after the run MISSED the oom event although OOMKilled=true; after a 2 s wait and `--until` = guest epoch + 1 it is +# seen. Pinned by test_events_window_ends_after_the_settle_wait. +OOMCHECK_SETTLE = 2 # Kernel, boot and firmware packages are the SLOW lane on the host whatever their origin (`11` C3, §5.2): a host # reboot is needed for them to take effect, and a bad one can stop the box from booting. HOST_SLOW_RE = re.compile(r"^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|" @@ -405,7 +423,7 @@ class Apply: def check_plan(self, plan): mode = plan.get("mode", "apply") - if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update"): + if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update", "oom-check"): raise Refused("R11", f"unknown mode {mode!r}") if mode == "agent_update": if plan.get("layer") != "host": @@ -427,6 +445,8 @@ class Apply: raise Refused("R11", "facts is a host-layer mode (it reads the host and the guest)") if mode == "live-restore-on" and layer != "guest": raise Refused("R11", "live-restore-on is a guest-layer mode") + if mode == "oom-check" and layer != "docker": + raise Refused("R11", "oom-check is a docker-layer mode (it checks the guest's Docker engine)") if plan.get("undo") and layer != "docker": raise Refused("R5", "an undo (downgrade) exists only for the Docker layer, inside a signed job") vmid = plan.get("vmid") @@ -951,6 +971,10 @@ class Apply: log = self.r.log if self.mode == "live-restore-on": return self.live_restore_on() + if self.mode == "oom-check": + # R-528: the check alone — no apt, no engine change; check_guest above still applies. + self.report["oom_check"] = self.oom_check() + return 0 self.who, self.allow_downgrade = ("fast", False) if self.layer == "docker" and self.mode == "apply": self.who, self.allow_downgrade = self.docker_authority(plan) @@ -990,8 +1014,87 @@ class Apply: self.report["docker_engine"] = out_v.strip() if rc_v == 0 and out_v.strip() else "unknown" self.report["reboot_scanned"] = "reboot_needed" in self.report self.report["health_after"] = self.health() + if self.layer == "docker" and self.mode == "apply": + # R-528 (`09` decision 157): does the engine report a memory kill? Reported only — never the outcome. + self.report["oom_check"] = self.oom_check() return 0 + # ---------- R-528: the memory-kill check ---------- + def oom_check(self): + """Run one throwaway container over its memory cap in the guest and read what the engine says about it. + Never raises: any failure becomes result "error". The container is ALWAYS removed (finally), and a failed + removal is named in the detail.""" + t = OOMCHECK_TIMEOUTS + res = {"result": "error", "oom_killed": False, "oom_event": False, "exit_code": None, "image": None, "detail": ""} + log = self.r.log + try: + rc, out, err = self.g(["docker", "inspect", "-f", "{{.Config.Image}}", "felhom-controller"], timeout=t["image"]) + except Exception as e: + rc, out, err = -1, "", str(e) + img = out.strip() if rc == 0 else "" + if not img or any(c.isspace() for c in img): + res["detail"] = f"the controller's image could not be read (rc={rc}): {(err or out).strip()[:200]}" + log(f"os-apply: OOM-CHECK error — {res['detail']}") + return res + res["image"] = img + name = f"{OOMCHECK_PREFIX}{os.getpid()}-{os.urandom(4).hex()}" + notes = [] + try: + t0 = self.guest_epoch() + if t0 is None: + raise RuntimeError("the guest clock could not be read") + rrc, rout, rerr = self.g(["docker", "run", "--name", name, "--pull", "never", "--network", "none", + "--memory", "64m", "--memory-swap", "64m", "--label", "felhom.oomcheck=1", + "--entrypoint", "sh", img, "-c", OOMCHECK_SCRIPT], timeout=t["run"]) + self.r.sleep(OOMCHECK_SETTLE) # the engine publishes the oom event a moment after the run returns (F1/F2) + t1 = self.guest_epoch() + if t1 is None: + t1 = t0 + t["run"] + OOMCHECK_SETTLE + 1 + irc, iout, ierr = self.g(["docker", "inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}", name], timeout=t["inspect"]) + if irc != 0: + raise RuntimeError(f"the check container could not be inspected (run rc={rrc}: {(rerr or rout).strip()[:120]}; " + f"inspect rc={irc}: {(ierr or iout).strip()[:120]})") + f = iout.split() + res["oom_killed"] = bool(f) and f[0] == "true" + try: + res["exit_code"] = int(f[1]) if len(f) > 1 else None + except ValueError: + res["exit_code"] = None + erc, eout, eerr = self.g(["docker", "events", "--since", str(t0 - 1), "--until", str(t1 + 1), + "--filter", f"container={name}", "--filter", "event=oom", + "--format", "{{.Action}}"], timeout=t["events"]) + if erc != 0: + notes.append(f"the event read failed (rc={erc}): {(eerr or eout).strip()[:120]}") + res["oom_event"] = erc == 0 and any(l.strip() == "oom" for l in eout.splitlines()) + if res["oom_killed"] and res["oom_event"]: + res["result"] = "pass" + notes.insert(0, "the engine reported the memory kill: OOMKilled=true and the oom event") + else: + res["result"] = "fail" + miss = [w for w, ok in (("OOMKilled=true", res["oom_killed"]), ("the oom event", res["oom_event"])) if not ok] + notes.insert(0, f"the engine did not report the memory kill: missing {' and '.join(miss)} (exit code {res['exit_code']})") + except Exception as e: + res["result"] = "error" + notes.insert(0, f"the check could not finish: {type(e).__name__}: {str(e)[:200]}") + finally: + try: + mrc, mout, merr = self.g(["docker", "rm", "-f", name], timeout=t["rm"]) + if mrc != 0 and "no such container" not in (merr + mout).lower(): + notes.append(f"the check container {name} could not be removed (rc={mrc}): {(merr or mout).strip()[:120]}") + except Exception as e: + notes.append(f"the check container {name} could not be removed: {type(e).__name__}: {str(e)[:120]}") + res["detail"] = "; ".join(notes) + log(f"os-apply: OOM-CHECK result={res['result']} oom_killed={res['oom_killed']} oom_event={res['oom_event']} " + f"exit={res['exit_code']} image={img} — {res['detail']}") + return res + + def guest_epoch(self): + rc, out, _ = self.g(["date", "+%s"], timeout=OOMCHECK_TIMEOUTS["clock"]) + try: + return int(out.strip()) if rc == 0 else None + except ValueError: + return None + def dpkg_state(self): """`dpkg --audit` AND dpkg's update journal, in ONE call (R-876, agent v0.145.0). A crash in the middle of an install can leave `/var/lib/dpkg/updates/` non-empty while `--audit` reads clean — measured on demo-hp diff --git a/configs/test_felhom_os_apply.py b/configs/test_felhom_os_apply.py index 318dd33..c9e1805 100644 --- a/configs/test_felhom_os_apply.py +++ b/configs/test_felhom_os_apply.py @@ -83,7 +83,7 @@ class Fake: return self.clock def sleep(self, s): - pass + self.sleeps = getattr(self, "sleeps", []) + [(len(self.calls), s)] # (calls made before it, seconds) def verify_sig(self, signers, key_id, ns, blob, sig): self.verified = (signers, key_id, ns, blob, sig) @@ -196,6 +196,27 @@ class Fake: return 0, self.engine + "\n", "" if cmd == "docker" and a[1:3] == ["ps", "-q"]: return 0, "".join(i + "\n" for i in self.ids), "" + # R-528: the memory-kill check's engine (oom_image None = unreadable; oom_state / oom_event the engine's answer) + if cmd == "date" and a[1:] == ["+%s"]: + # each read is 3 s later than the last, so the order of the reads is visible in the values + self.oom_epochs = getattr(self, "oom_epochs", []) + [int(self.clock) + 3 * len(getattr(self, "oom_epochs", []))] + return 0, f"{self.oom_epochs[-1]}\n", "" + if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.Config.Image}}"]: + img = getattr(self, "oom_image", "gitea.dooplex.hu/admin/felhom-controller:0.300.0") + return (0, img + "\n", "") if img is not None else (1, "", "Error: No such object: felhom-controller") + if cmd == "docker" and a[1] == "run": + self.oom_runs = getattr(self, "oom_runs", []) + [a] + return 137, "", "" + if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}"]: + if getattr(self, "oom_inspect_raises", False): + raise subprocess.TimeoutExpired(a, 10) + return 0, getattr(self, "oom_state", "true 137") + "\n", "" + if cmd == "docker" and a[1] == "events": + self.oom_events_argv = a + return 0, ("oom\n" if getattr(self, "oom_event", True) else ""), "" + if cmd == "docker" and a[1:3] == ["rm", "-f"]: + self.oom_removed = getattr(self, "oom_removed", []) + a[3:] + return 0, a[3] + "\n", "" if cmd == "docker" and a[1] == "inspect": mounts = {"aaa111": "/felhom-controller|/var/run/docker.sock;/app/data;", "bbb222": "/app|/data;"} return 0, mounts.get(a[-1], "/other|;") + "\n", "" @@ -997,8 +1018,6 @@ class RealSignatureCheck(unittest.TestCase): self.assertNotEqual(r.verify_sig(self.signers, "someone-else", "felhom-op-v1", blob, sig), 0) self.assertNotEqual(r.verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob, self.sign(blob, ns="other-ns")), 0) -if __name__ == "__main__": - unittest.main() class UnsentReport(unittest.TestCase): @@ -1179,3 +1198,131 @@ class CrashLeftTheJournal(unittest.TestCase): self.assertEqual(rc, 0, rep) self.assertTrue(any("INTERRUPTED" in l for l in f.logs), f.logs) self.assertTrue(any(l.startswith("os-apply: REPAIR ") and l.endswith("forced") for l in f.logs), f.logs) + + +class OOMCheck(unittest.TestCase): + """R-528 (`09` decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill + (OOMKilled=true AND the `oom` event). Reported only; the hub decides. Red-proofs: audits/readback-2026-10-07/F/.""" + + def apply(self, **kw): + f = docker_fake(signed=signed_job()) + for k, v in kw.items(): + setattr(f, k, v) + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + return f, rep + + def test_pass_when_oomkilled_and_the_event(self): + f, rep = self.apply() + oc = rep["oom_check"] + self.assertEqual(oc["result"], "pass", oc) + self.assertEqual((oc["oom_killed"], oc["oom_event"], oc["exit_code"]), (True, True, 137)) + self.assertEqual(oc["image"], "gitea.dooplex.hu/admin/felhom-controller:0.300.0") + self.assertEqual(sorted(oc), ["detail", "exit_code", "image", "oom_event", "oom_killed", "result"]) + run_argv = f.oom_runs[0] + name = run_argv[run_argv.index("--name") + 1] + self.assertTrue(re.match(r"^felhom-oomcheck-[0-9]+-[0-9a-f]{8}$", name), name) + for flag, val in (("--pull", "never"), ("--network", "none"), ("--memory", "64m"), ("--memory-swap", "64m"), + ("--label", "felhom.oomcheck=1"), ("--entrypoint", "sh")): + self.assertEqual(run_argv[run_argv.index(flag) + 1], val, flag) + self.assertNotIn("-v", run_argv) + self.assertNotIn("-p", run_argv) + self.assertEqual(run_argv[-3:], ["gitea.dooplex.hu/admin/felhom-controller:0.300.0", "-c", osapply.OOMCHECK_SCRIPT]) + self.assertIn(f"container={name}", f.oom_events_argv) + self.assertIn("event=oom", f.oom_events_argv) + self.assertEqual(f.oom_removed, [name], "the check container must be removed") + self.assertTrue(rep["health_after"], "health is read before the check") + # bounded: every call has a timeout and the clock is read twice — the worst case stays within 90 s + self.assertLessEqual(sum(osapply.OOMCHECK_TIMEOUTS.values()) + osapply.OOMCHECK_TIMEOUTS["clock"] + + osapply.OOMCHECK_SETTLE, 90) + + def test_events_window_ends_after_the_settle_wait(self): + # Measured (F1/F2): an --until taken right after the run missed the oom event. The window must end after a + # wait of >= 2 s that comes AFTER the run, at the guest epoch read after that wait, + 1. + f, rep = self.apply() + run_i = next(i for i, c in enumerate(f.calls) if c[-1][:2] == ["docker", "run"]) + date_i = [i for i, c in enumerate(f.calls) if c[-1] == ["date", "+%s"]] + waits = [(i, s) for i, s in getattr(f, "sleeps", []) if i > run_i] + self.assertTrue(waits and waits[0][1] >= 2, f"no settle wait after the run: {getattr(f, 'sleeps', None)}") + self.assertTrue(date_i[-1] >= waits[0][0], "the end epoch must be read after the wait") + ev = f.oom_events_argv + since, until = int(ev[ev.index("--since") + 1]), int(ev[ev.index("--until") + 1]) + self.assertEqual(until, f.oom_epochs[-1] + 1, "until = the guest epoch read after the wait, + 1") + self.assertGreater(until, f.oom_epochs[0] + 1) + self.assertLess(since, f.oom_epochs[0] + 1) + + def test_oomkilled_false_is_fail(self): + f, rep = self.apply(oom_state="false 0") + oc = rep["oom_check"] + self.assertEqual(oc["result"], "fail", oc) + self.assertIn("OOMKilled=true", oc["detail"]) + self.assertTrue(oc["oom_event"]) + self.assertEqual(len(f.oom_removed), 1) + + def test_no_event_is_fail(self): + f, rep = self.apply(oom_event=False) + oc = rep["oom_check"] + self.assertEqual(oc["result"], "fail", oc) + self.assertIn("the oom event", oc["detail"]) + self.assertTrue(oc["oom_killed"]) + + def test_image_unreadable_is_error(self): + f, rep = self.apply(oom_image=None) + oc = rep["oom_check"] + self.assertEqual(oc["result"], "error", oc) + self.assertIsNone(oc["image"]) + self.assertIn("image could not be read", oc["detail"]) + self.assertFalse(hasattr(f, "oom_runs"), "no container is started without an image") + + def test_container_removed_even_when_inspect_raises(self): + f, rep = self.apply(oom_inspect_raises=True) + oc = rep["oom_check"] + self.assertEqual(oc["result"], "error", oc) + self.assertEqual(len(f.oom_removed), 1, "the container must be removed even when inspect raised") + self.assertTrue(f.oom_removed[0].startswith(osapply.OOMCHECK_PREFIX)) + self.assertNotIn("failed", rep, "the check never turns the step into a failure") + + def test_never_on_guest_or_host_or_in_health_mode(self): + g = Fake() + rc, rep = run(g) + self.assertEqual(rc, 0, rep) + h = Fake() + h.plan["layer"] = "host" + h.plan["packages"] = [{"name": "bash", "version": "5.2.37-2+b10", "origin": "Debian"}] + h.installed["bash"] = "5.2.37-2+b9" + h.live["bash"] = {"5.2.37-2+b10"} + rc_h, rep_h = run(h) + self.assertEqual(rc_h, 0, rep_h) + d = docker_fake(signed=signed_job()) + d.plan["mode"] = "health" + rc_d, rep_d = run(d) + self.assertEqual(rc_d, 0, rep_d) + for f, r in ((g, rep), (h, rep_h), (d, rep_d)): + self.assertNotIn("oom_check", r) + self.assertFalse(hasattr(f, "oom_runs"), r.get("layer")) + self.assertFalse(any(c[-1][:2] == ["docker", "run"] for c in f.calls)) + + def test_mode_oom_check_runs_only_the_check(self): + f = docker_fake() # no authority: the check changes nothing, so it needs none + f.plan["mode"], f.plan["packages"] = "oom-check", [] + rc, rep = run(f) + self.assertEqual(rc, 0, rep) + self.assertEqual(rep["oom_check"]["result"], "pass", rep) + self.assertFalse(any("apt-get" in c[-1] or "dpkg-query" in c[-1] for c in f.calls), f.calls) + self.assertEqual(len(f.oom_removed), 1) + + def test_mode_oom_check_keeps_the_refusals(self): + f = docker_fake() + f.plan["mode"], f.plan["packages"] = "oom-check", [] + f.files["/etc/pve/lxc/9201.conf"] = "arch: amd64\n" + rc, rep = run(f) + self.assertEqual((rc, rep["refused"]["code"]), (2, "R10"), rep) + self.assertFalse(hasattr(f, "oom_runs")) + g = Fake() + g.plan["mode"] = "oom-check" + rc, rep = run(g) + self.assertEqual((rc, rep["refused"]["code"]), (2, "R11"), rep) + + +if __name__ == "__main__": + unittest.main() diff --git a/internal/osupdate/leg.go b/internal/osupdate/leg.go index 9c675a2..6d18b4a 100644 --- a/internal/osupdate/leg.go +++ b/internal/osupdate/leg.go @@ -106,6 +106,9 @@ type WrapperReport struct { LiveRestore json.RawMessage `json:"live_restore"` Facts json.RawMessage `json:"facts"` Bundle json.RawMessage `json:"bundle"` // the config bundle's result (R-840, mode "bundle") + // OOMCheck (R-528, `09` decision 157): the docker layer's memory-kill check, {result, oom_killed, oom_event, + // exit_code, image, detail}. Carried to the hub UNCHANGED; the agent never reads it. + OOMCheck json.RawMessage `json:"oom_check"` // R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the // agent process that started the pass. ReleaseID / VMID were always in the report. RunID string `json:"run_id"` @@ -143,6 +146,9 @@ type Report struct { DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade) + // OOMCheck: docker layer — the wrapper's oom_check object, byte-for-byte (R-528; the hub decides approval on it). + // Pinned by TestDocker_OOMCheckReachesTheHubUnchanged and TestR868_KeptCopyCarriesTheOOMCheck. + OOMCheck json.RawMessage `json:"oom_check,omitempty"` unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it } @@ -630,6 +636,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg } rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo + rep.OOMCheck = rawOrNil(wr.OOMCheck) if rep.Outcome == "" { switch { case rep.Mode == "inventory" && !blk.Enabled: @@ -707,6 +714,14 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg return l.finish(ctx, lg, rep) } +// rawOrNil: a wrapper field that is absent or JSON null stays out of the hub report (omitempty). +func rawOrNil(m json.RawMessage) json.RawMessage { + if len(m) == 0 || string(m) == "null" { + return nil + } + return m +} + func onlyDocker(in []Package) []Package { var out []Package for _, p := range in { diff --git a/internal/osupdate/leg_test.go b/internal/osupdate/leg_test.go index d5a8e15..739b35e 100644 --- a/internal/osupdate/leg_test.go +++ b/internal/osupdate/leg_test.go @@ -98,9 +98,13 @@ func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, ar return f.Run(ctx, name, args...) } -type fakeHub struct{ reports []Report } +type fakeHub struct { + reports []Report + bodies [][]byte // the exact bytes posted (R-528: the oom_check object must arrive unchanged) +} func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error { + h.bodies = append(h.bodies, append([]byte(nil), body...)) var r Report json.Unmarshal(body, &r) h.reports = append(h.reports, r) @@ -504,3 +508,42 @@ func TestHealthVerdict_ControllerBlindToDockerFails(t *testing.T) { t.Fatal("an older wrapper (no field) must not fail") } } + +// R-528 (`09` decision 157): the wrapper's oom_check object reaches the hub's docker report byte-for-byte; the guest +// and host reports carry none. COMPANION RED-PROOF: drop `rep.OOMCheck = rawOrNil(wr.OOMCheck)` in runLayer → "no +// oom_check in the docker report". +const oomCheckWire = `{"detail":"the engine reported the memory kill: OOMKilled=true and the oom event","exit_code":137,"image":"gitea.dooplex.hu/admin/felhom-controller:0.300.0","oom_event":true,"oom_killed":true,"result":"pass"}` + +func TestDocker_OOMCheckReachesTheHubUnchanged(t *testing.T) { + w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: { + Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, + DockerEngine: "29.8.2", Authority: "ring0", OOMCheck: json.RawMessage(oomCheckWire)}}} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) + l.Run(context.Background(), 9201, "night") + found := false + for _, b := range h.bodies { + var m map[string]json.RawMessage + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + var layer string + json.Unmarshal(m["layer"], &layer) + oc, has := m["oom_check"] + if layer != LayerDocker { + if has { + t.Fatalf("the %s report carries an oom_check: %s", layer, oc) + } + continue + } + found = true + if !has { + t.Fatalf("no oom_check in the docker report: %s", b) + } + if string(oc) != oomCheckWire { + t.Fatalf("oom_check changed on the way:\n got %s\nwant %s", oc, oomCheckWire) + } + } + if !found { + t.Fatalf("no docker report posted: %s", calls(w)) + } +} diff --git a/internal/osupdate/unsent.go b/internal/osupdate/unsent.go index c647a25..5a23c5c 100644 --- a/internal/osupdate/unsent.go +++ b/internal/osupdate/unsent.go @@ -144,6 +144,7 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string) } rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo + rep.OOMCheck = rawOrNil(wr.OOMCheck) wantEngine := "" for _, u := range wr.Upgraded { if u.Name == "docker-ce" { diff --git a/internal/osupdate/unsent_test.go b/internal/osupdate/unsent_test.go index 774e44d..595c1dc 100644 --- a/internal/osupdate/unsent_test.go +++ b/internal/osupdate/unsent_test.go @@ -150,3 +150,24 @@ func must(t *testing.T, err error) { t.Fatal(err) } } + +// R-528: a kept docker report (the agent was killed) still carries the oom_check object to the hub unchanged. +// COMPANION RED-PROOF: drop `rep.OOMCheck = rawOrNil(wr.OOMCheck)` in reportFromKept → "oom_check lost". +func TestR868_KeptCopyCarriesTheOOMCheck(t *testing.T) { + w := &fakeWrapper{t: t} + l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) + ring := 0 + kept := WrapperReport{Mode: "apply", Layer: LayerDocker, RunID: "20261007T020000Z", Trigger: "night", Ring: &ring, VMID: 9201, + ReleaseID: "ring0-20261007T020000Z", HealthBefore: guestOK(), HealthAfter: guestOK(), DockerEngine: "29.8.2", + Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, OOMCheck: json.RawMessage(oomCheckWire)} + b, _ := json.Marshal(kept) + must(t, os.WriteFile(reportFile(l.PlanDir, kept.RunID, LayerDocker, "apply"), b, 0o600)) + if n := l.SendUnsent(context.Background()); n != 1 || len(h.bodies) != 1 { + t.Fatalf("sent %d, bodies %d", n, len(h.bodies)) + } + var m map[string]json.RawMessage + must(t, json.Unmarshal(h.bodies[0], &m)) + if string(m["oom_check"]) != oomCheckWire { + t.Fatalf("oom_check lost or changed: %q", m["oom_check"]) + } +}