R-528 (09 decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill

felhom-os-apply: a docker-layer apply runs oom_check() after health_after and reports
"oom_check": {result pass|fail|error, oom_killed, oom_event, exit_code, image, detail}.
One throwaway container (the controller's image, --pull never, --network none, 64m cap,
label felhom.oomcheck=1) runs dd bs=200M; pass only with OOMKilled=true AND the oom event
(read after a 2 s settle, --until = guest epoch + 1: measured on demo-hp, an --until taken
right after the run missed the event). docker rm -f always runs in a finally; every call
is bounded (<= 90 s). It never changes the step's outcome or health. New wrapper-only mode
"oom-check" (docker layer) runs the check alone; check_guest etc. still apply.

Agent: WrapperReport/Report gain OOMCheck (json:"oom_check"), copied unchanged in runLayer
and in the kept-copy path.

Tests: 9 wrapper tests + 2 Go tests, each red-proofed (audits/readback-2026-10-07/F/red-*.txt).
Also: test_felhom_os_apply.py's `if __name__` sat mid-file, so 11 tests (UnsentReport,
SaveReportOnDisk, AgentDiesMidPass, CrashLeftTheJournal) never ran as a script or from
TestWrapperSuite; moved to the end (they pass).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-06 19:15:06 +02:00
parent de812bc027
commit acccb66bd3
6 changed files with 335 additions and 5 deletions
+104 -1
View File
@@ -32,6 +32,14 @@
# held packages, kernel taint, the crash guard; guest Debian, Docker engine, containerd, live-restore.
# live-restore-on (v0.142.0, layer guest) the ONE-TIME step of `09` decision 87: merge `"live-restore": true`
# into the guest's /etc/docker/daemon.json and `systemctl reload docker`. NEVER a restart (R-835).
# oom-check (R-528, `09` decision 157; layer docker, lane slow) ONLY the memory-kill check below: no apt, no engine
# change, no authority needed. Wrapper-only — the agent never writes this plan; run it by hand as root.
# R-528 (`09` decision 157): a docker-layer APPLY also runs `oom_check()` after health_after and reports it as
# "oom_check": {"result": "pass"|"fail"|"error", "oom_killed": bool, "oom_event": bool, "exit_code": int|null,
# "image": str|null, "detail": str}
# — one throwaway container (the controller's own image, no network / volume / port, 64 MB cap) is made to exceed its
# memory; "pass" only when the engine says OOMKilled=true AND emits the `oom` event. It never changes the step's
# outcome or health: the hub decides whether the engine set can be approved. Pinned by the OOMCheck tests.
# Output: log lines on stderr and the journal (tag felhom-os-apply); the LAST stdout line is
# OSAPPLY-REPORT <one JSON object>
# which is what the agent parses. Exit 0 = done; 2 = refused (nothing changed); 3 = failed during install.
@@ -68,6 +76,16 @@ JOURNAL_MARK = "@@FELHOM-DPKG-JOURNAL@@"
DPKG_STATE_SCRIPT = "dpkg --audit; echo " + JOURNAL_MARK + "; ls -A /var/lib/dpkg/updates 2>/dev/null; true"
# The installer's ROOT-OWNED record (felhom-host-install.sh `state_set mode`); the agent cannot write it.
INSTALL_STATE = "/var/lib/felhom-install/state.json"
# R-528: the memory-kill check (oom_check). One 200 MB block under a 64 MB cap: measured on Docker 29.8.2 to be
# OOM-killed with OOMKilled=true and an `oom` event. Every call is bounded: timeouts (the clock read twice) + the
# settle wait stay within 90 s.
OOMCHECK_PREFIX = "felhom-oomcheck-"
OOMCHECK_SCRIPT = "dd if=/dev/zero of=/dev/null bs=200M count=1"
OOMCHECK_TIMEOUTS = {"image": 10, "clock": 5, "run": 30, "inspect": 10, "events": 10, "rm": 15}
# Measured on demo-hp 9201 (Docker 29.8.2, 2026-10-06, audits/readback-2026-10-07/F/F1, F2): an `--until` taken right
# after the run MISSED the oom event although OOMKilled=true; after a 2 s wait and `--until` = guest epoch + 1 it is
# seen. Pinned by test_events_window_ends_after_the_settle_wait.
OOMCHECK_SETTLE = 2
# Kernel, boot and firmware packages are the SLOW lane on the host whatever their origin (`11` C3, §5.2): a host
# reboot is needed for them to take effect, and a bad one can stop the box from booting.
HOST_SLOW_RE = re.compile(r"^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|"
@@ -405,7 +423,7 @@ class Apply:
def check_plan(self, plan):
mode = plan.get("mode", "apply")
if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update"):
if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update", "oom-check"):
raise Refused("R11", f"unknown mode {mode!r}")
if mode == "agent_update":
if plan.get("layer") != "host":
@@ -427,6 +445,8 @@ class Apply:
raise Refused("R11", "facts is a host-layer mode (it reads the host and the guest)")
if mode == "live-restore-on" and layer != "guest":
raise Refused("R11", "live-restore-on is a guest-layer mode")
if mode == "oom-check" and layer != "docker":
raise Refused("R11", "oom-check is a docker-layer mode (it checks the guest's Docker engine)")
if plan.get("undo") and layer != "docker":
raise Refused("R5", "an undo (downgrade) exists only for the Docker layer, inside a signed job")
vmid = plan.get("vmid")
@@ -951,6 +971,10 @@ class Apply:
log = self.r.log
if self.mode == "live-restore-on":
return self.live_restore_on()
if self.mode == "oom-check":
# R-528: the check alone — no apt, no engine change; check_guest above still applies.
self.report["oom_check"] = self.oom_check()
return 0
self.who, self.allow_downgrade = ("fast", False)
if self.layer == "docker" and self.mode == "apply":
self.who, self.allow_downgrade = self.docker_authority(plan)
@@ -990,8 +1014,87 @@ class Apply:
self.report["docker_engine"] = out_v.strip() if rc_v == 0 and out_v.strip() else "unknown"
self.report["reboot_scanned"] = "reboot_needed" in self.report
self.report["health_after"] = self.health()
if self.layer == "docker" and self.mode == "apply":
# R-528 (`09` decision 157): does the engine report a memory kill? Reported only — never the outcome.
self.report["oom_check"] = self.oom_check()
return 0
# ---------- R-528: the memory-kill check ----------
def oom_check(self):
"""Run one throwaway container over its memory cap in the guest and read what the engine says about it.
Never raises: any failure becomes result "error". The container is ALWAYS removed (finally), and a failed
removal is named in the detail."""
t = OOMCHECK_TIMEOUTS
res = {"result": "error", "oom_killed": False, "oom_event": False, "exit_code": None, "image": None, "detail": ""}
log = self.r.log
try:
rc, out, err = self.g(["docker", "inspect", "-f", "{{.Config.Image}}", "felhom-controller"], timeout=t["image"])
except Exception as e:
rc, out, err = -1, "", str(e)
img = out.strip() if rc == 0 else ""
if not img or any(c.isspace() for c in img):
res["detail"] = f"the controller's image could not be read (rc={rc}): {(err or out).strip()[:200]}"
log(f"os-apply: OOM-CHECK error — {res['detail']}")
return res
res["image"] = img
name = f"{OOMCHECK_PREFIX}{os.getpid()}-{os.urandom(4).hex()}"
notes = []
try:
t0 = self.guest_epoch()
if t0 is None:
raise RuntimeError("the guest clock could not be read")
rrc, rout, rerr = self.g(["docker", "run", "--name", name, "--pull", "never", "--network", "none",
"--memory", "64m", "--memory-swap", "64m", "--label", "felhom.oomcheck=1",
"--entrypoint", "sh", img, "-c", OOMCHECK_SCRIPT], timeout=t["run"])
self.r.sleep(OOMCHECK_SETTLE) # the engine publishes the oom event a moment after the run returns (F1/F2)
t1 = self.guest_epoch()
if t1 is None:
t1 = t0 + t["run"] + OOMCHECK_SETTLE + 1
irc, iout, ierr = self.g(["docker", "inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}", name], timeout=t["inspect"])
if irc != 0:
raise RuntimeError(f"the check container could not be inspected (run rc={rrc}: {(rerr or rout).strip()[:120]}; "
f"inspect rc={irc}: {(ierr or iout).strip()[:120]})")
f = iout.split()
res["oom_killed"] = bool(f) and f[0] == "true"
try:
res["exit_code"] = int(f[1]) if len(f) > 1 else None
except ValueError:
res["exit_code"] = None
erc, eout, eerr = self.g(["docker", "events", "--since", str(t0 - 1), "--until", str(t1 + 1),
"--filter", f"container={name}", "--filter", "event=oom",
"--format", "{{.Action}}"], timeout=t["events"])
if erc != 0:
notes.append(f"the event read failed (rc={erc}): {(eerr or eout).strip()[:120]}")
res["oom_event"] = erc == 0 and any(l.strip() == "oom" for l in eout.splitlines())
if res["oom_killed"] and res["oom_event"]:
res["result"] = "pass"
notes.insert(0, "the engine reported the memory kill: OOMKilled=true and the oom event")
else:
res["result"] = "fail"
miss = [w for w, ok in (("OOMKilled=true", res["oom_killed"]), ("the oom event", res["oom_event"])) if not ok]
notes.insert(0, f"the engine did not report the memory kill: missing {' and '.join(miss)} (exit code {res['exit_code']})")
except Exception as e:
res["result"] = "error"
notes.insert(0, f"the check could not finish: {type(e).__name__}: {str(e)[:200]}")
finally:
try:
mrc, mout, merr = self.g(["docker", "rm", "-f", name], timeout=t["rm"])
if mrc != 0 and "no such container" not in (merr + mout).lower():
notes.append(f"the check container {name} could not be removed (rc={mrc}): {(merr or mout).strip()[:120]}")
except Exception as e:
notes.append(f"the check container {name} could not be removed: {type(e).__name__}: {str(e)[:120]}")
res["detail"] = "; ".join(notes)
log(f"os-apply: OOM-CHECK result={res['result']} oom_killed={res['oom_killed']} oom_event={res['oom_event']} "
f"exit={res['exit_code']} image={img} — {res['detail']}")
return res
def guest_epoch(self):
rc, out, _ = self.g(["date", "+%s"], timeout=OOMCHECK_TIMEOUTS["clock"])
try:
return int(out.strip()) if rc == 0 else None
except ValueError:
return None
def dpkg_state(self):
"""`dpkg --audit` AND dpkg's update journal, in ONE call (R-876, agent v0.145.0). A crash in the middle of an
install can leave `/var/lib/dpkg/updates/` non-empty while `--audit` reads clean — measured on demo-hp
+150 -3
View File
@@ -83,7 +83,7 @@ class Fake:
return self.clock
def sleep(self, s):
pass
self.sleeps = getattr(self, "sleeps", []) + [(len(self.calls), s)] # (calls made before it, seconds)
def verify_sig(self, signers, key_id, ns, blob, sig):
self.verified = (signers, key_id, ns, blob, sig)
@@ -196,6 +196,27 @@ class Fake:
return 0, self.engine + "\n", ""
if cmd == "docker" and a[1:3] == ["ps", "-q"]:
return 0, "".join(i + "\n" for i in self.ids), ""
# R-528: the memory-kill check's engine (oom_image None = unreadable; oom_state / oom_event the engine's answer)
if cmd == "date" and a[1:] == ["+%s"]:
# each read is 3 s later than the last, so the order of the reads is visible in the values
self.oom_epochs = getattr(self, "oom_epochs", []) + [int(self.clock) + 3 * len(getattr(self, "oom_epochs", []))]
return 0, f"{self.oom_epochs[-1]}\n", ""
if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.Config.Image}}"]:
img = getattr(self, "oom_image", "gitea.dooplex.hu/admin/felhom-controller:0.300.0")
return (0, img + "\n", "") if img is not None else (1, "", "Error: No such object: felhom-controller")
if cmd == "docker" and a[1] == "run":
self.oom_runs = getattr(self, "oom_runs", []) + [a]
return 137, "", ""
if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}"]:
if getattr(self, "oom_inspect_raises", False):
raise subprocess.TimeoutExpired(a, 10)
return 0, getattr(self, "oom_state", "true 137") + "\n", ""
if cmd == "docker" and a[1] == "events":
self.oom_events_argv = a
return 0, ("oom\n" if getattr(self, "oom_event", True) else ""), ""
if cmd == "docker" and a[1:3] == ["rm", "-f"]:
self.oom_removed = getattr(self, "oom_removed", []) + a[3:]
return 0, a[3] + "\n", ""
if cmd == "docker" and a[1] == "inspect":
mounts = {"aaa111": "/felhom-controller|/var/run/docker.sock;/app/data;", "bbb222": "/app|/data;"}
return 0, mounts.get(a[-1], "/other|;") + "\n", ""
@@ -997,8 +1018,6 @@ class RealSignatureCheck(unittest.TestCase):
self.assertNotEqual(r.verify_sig(self.signers, "someone-else", "felhom-op-v1", blob, sig), 0)
self.assertNotEqual(r.verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob, self.sign(blob, ns="other-ns")), 0)
if __name__ == "__main__":
unittest.main()
class UnsentReport(unittest.TestCase):
@@ -1179,3 +1198,131 @@ class CrashLeftTheJournal(unittest.TestCase):
self.assertEqual(rc, 0, rep)
self.assertTrue(any("INTERRUPTED" in l for l in f.logs), f.logs)
self.assertTrue(any(l.startswith("os-apply: REPAIR ") and l.endswith("forced") for l in f.logs), f.logs)
class OOMCheck(unittest.TestCase):
"""R-528 (`09` decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill
(OOMKilled=true AND the `oom` event). Reported only; the hub decides. Red-proofs: audits/readback-2026-10-07/F/."""
def apply(self, **kw):
f = docker_fake(signed=signed_job())
for k, v in kw.items():
setattr(f, k, v)
rc, rep = run(f)
self.assertEqual(rc, 0, rep)
return f, rep
def test_pass_when_oomkilled_and_the_event(self):
f, rep = self.apply()
oc = rep["oom_check"]
self.assertEqual(oc["result"], "pass", oc)
self.assertEqual((oc["oom_killed"], oc["oom_event"], oc["exit_code"]), (True, True, 137))
self.assertEqual(oc["image"], "gitea.dooplex.hu/admin/felhom-controller:0.300.0")
self.assertEqual(sorted(oc), ["detail", "exit_code", "image", "oom_event", "oom_killed", "result"])
run_argv = f.oom_runs[0]
name = run_argv[run_argv.index("--name") + 1]
self.assertTrue(re.match(r"^felhom-oomcheck-[0-9]+-[0-9a-f]{8}$", name), name)
for flag, val in (("--pull", "never"), ("--network", "none"), ("--memory", "64m"), ("--memory-swap", "64m"),
("--label", "felhom.oomcheck=1"), ("--entrypoint", "sh")):
self.assertEqual(run_argv[run_argv.index(flag) + 1], val, flag)
self.assertNotIn("-v", run_argv)
self.assertNotIn("-p", run_argv)
self.assertEqual(run_argv[-3:], ["gitea.dooplex.hu/admin/felhom-controller:0.300.0", "-c", osapply.OOMCHECK_SCRIPT])
self.assertIn(f"container={name}", f.oom_events_argv)
self.assertIn("event=oom", f.oom_events_argv)
self.assertEqual(f.oom_removed, [name], "the check container must be removed")
self.assertTrue(rep["health_after"], "health is read before the check")
# bounded: every call has a timeout and the clock is read twice — the worst case stays within 90 s
self.assertLessEqual(sum(osapply.OOMCHECK_TIMEOUTS.values()) + osapply.OOMCHECK_TIMEOUTS["clock"]
+ osapply.OOMCHECK_SETTLE, 90)
def test_events_window_ends_after_the_settle_wait(self):
# Measured (F1/F2): an --until taken right after the run missed the oom event. The window must end after a
# wait of >= 2 s that comes AFTER the run, at the guest epoch read after that wait, + 1.
f, rep = self.apply()
run_i = next(i for i, c in enumerate(f.calls) if c[-1][:2] == ["docker", "run"])
date_i = [i for i, c in enumerate(f.calls) if c[-1] == ["date", "+%s"]]
waits = [(i, s) for i, s in getattr(f, "sleeps", []) if i > run_i]
self.assertTrue(waits and waits[0][1] >= 2, f"no settle wait after the run: {getattr(f, 'sleeps', None)}")
self.assertTrue(date_i[-1] >= waits[0][0], "the end epoch must be read after the wait")
ev = f.oom_events_argv
since, until = int(ev[ev.index("--since") + 1]), int(ev[ev.index("--until") + 1])
self.assertEqual(until, f.oom_epochs[-1] + 1, "until = the guest epoch read after the wait, + 1")
self.assertGreater(until, f.oom_epochs[0] + 1)
self.assertLess(since, f.oom_epochs[0] + 1)
def test_oomkilled_false_is_fail(self):
f, rep = self.apply(oom_state="false 0")
oc = rep["oom_check"]
self.assertEqual(oc["result"], "fail", oc)
self.assertIn("OOMKilled=true", oc["detail"])
self.assertTrue(oc["oom_event"])
self.assertEqual(len(f.oom_removed), 1)
def test_no_event_is_fail(self):
f, rep = self.apply(oom_event=False)
oc = rep["oom_check"]
self.assertEqual(oc["result"], "fail", oc)
self.assertIn("the oom event", oc["detail"])
self.assertTrue(oc["oom_killed"])
def test_image_unreadable_is_error(self):
f, rep = self.apply(oom_image=None)
oc = rep["oom_check"]
self.assertEqual(oc["result"], "error", oc)
self.assertIsNone(oc["image"])
self.assertIn("image could not be read", oc["detail"])
self.assertFalse(hasattr(f, "oom_runs"), "no container is started without an image")
def test_container_removed_even_when_inspect_raises(self):
f, rep = self.apply(oom_inspect_raises=True)
oc = rep["oom_check"]
self.assertEqual(oc["result"], "error", oc)
self.assertEqual(len(f.oom_removed), 1, "the container must be removed even when inspect raised")
self.assertTrue(f.oom_removed[0].startswith(osapply.OOMCHECK_PREFIX))
self.assertNotIn("failed", rep, "the check never turns the step into a failure")
def test_never_on_guest_or_host_or_in_health_mode(self):
g = Fake()
rc, rep = run(g)
self.assertEqual(rc, 0, rep)
h = Fake()
h.plan["layer"] = "host"
h.plan["packages"] = [{"name": "bash", "version": "5.2.37-2+b10", "origin": "Debian"}]
h.installed["bash"] = "5.2.37-2+b9"
h.live["bash"] = {"5.2.37-2+b10"}
rc_h, rep_h = run(h)
self.assertEqual(rc_h, 0, rep_h)
d = docker_fake(signed=signed_job())
d.plan["mode"] = "health"
rc_d, rep_d = run(d)
self.assertEqual(rc_d, 0, rep_d)
for f, r in ((g, rep), (h, rep_h), (d, rep_d)):
self.assertNotIn("oom_check", r)
self.assertFalse(hasattr(f, "oom_runs"), r.get("layer"))
self.assertFalse(any(c[-1][:2] == ["docker", "run"] for c in f.calls))
def test_mode_oom_check_runs_only_the_check(self):
f = docker_fake() # no authority: the check changes nothing, so it needs none
f.plan["mode"], f.plan["packages"] = "oom-check", []
rc, rep = run(f)
self.assertEqual(rc, 0, rep)
self.assertEqual(rep["oom_check"]["result"], "pass", rep)
self.assertFalse(any("apt-get" in c[-1] or "dpkg-query" in c[-1] for c in f.calls), f.calls)
self.assertEqual(len(f.oom_removed), 1)
def test_mode_oom_check_keeps_the_refusals(self):
f = docker_fake()
f.plan["mode"], f.plan["packages"] = "oom-check", []
f.files["/etc/pve/lxc/9201.conf"] = "arch: amd64\n"
rc, rep = run(f)
self.assertEqual((rc, rep["refused"]["code"]), (2, "R10"), rep)
self.assertFalse(hasattr(f, "oom_runs"))
g = Fake()
g.plan["mode"] = "oom-check"
rc, rep = run(g)
self.assertEqual((rc, rep["refused"]["code"]), (2, "R11"), rep)
if __name__ == "__main__":
unittest.main()
+15
View File
@@ -106,6 +106,9 @@ type WrapperReport struct {
LiveRestore json.RawMessage `json:"live_restore"`
Facts json.RawMessage `json:"facts"`
Bundle json.RawMessage `json:"bundle"` // the config bundle's result (R-840, mode "bundle")
// OOMCheck (R-528, `09` decision 157): the docker layer's memory-kill check, {result, oom_killed, oom_event,
// exit_code, image, detail}. Carried to the hub UNCHANGED; the agent never reads it.
OOMCheck json.RawMessage `json:"oom_check"`
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
// agent process that started the pass. ReleaseID / VMID were always in the report.
RunID string `json:"run_id"`
@@ -143,6 +146,9 @@ type Report struct {
DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade)
// OOMCheck: docker layer — the wrapper's oom_check object, byte-for-byte (R-528; the hub decides approval on it).
// Pinned by TestDocker_OOMCheckReachesTheHubUnchanged and TestR868_KeptCopyCarriesTheOOMCheck.
OOMCheck json.RawMessage `json:"oom_check,omitempty"`
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
}
@@ -630,6 +636,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
rep.OOMCheck = rawOrNil(wr.OOMCheck)
if rep.Outcome == "" {
switch {
case rep.Mode == "inventory" && !blk.Enabled:
@@ -707,6 +714,14 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
return l.finish(ctx, lg, rep)
}
// rawOrNil: a wrapper field that is absent or JSON null stays out of the hub report (omitempty).
func rawOrNil(m json.RawMessage) json.RawMessage {
if len(m) == 0 || string(m) == "null" {
return nil
}
return m
}
func onlyDocker(in []Package) []Package {
var out []Package
for _, p := range in {
+44 -1
View File
@@ -98,9 +98,13 @@ func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, ar
return f.Run(ctx, name, args...)
}
type fakeHub struct{ reports []Report }
type fakeHub struct {
reports []Report
bodies [][]byte // the exact bytes posted (R-528: the oom_check object must arrive unchanged)
}
func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
h.bodies = append(h.bodies, append([]byte(nil), body...))
var r Report
json.Unmarshal(body, &r)
h.reports = append(h.reports, r)
@@ -504,3 +508,42 @@ func TestHealthVerdict_ControllerBlindToDockerFails(t *testing.T) {
t.Fatal("an older wrapper (no field) must not fail")
}
}
// R-528 (`09` decision 157): the wrapper's oom_check object reaches the hub's docker report byte-for-byte; the guest
// and host reports carry none. COMPANION RED-PROOF: drop `rep.OOMCheck = rawOrNil(wr.OOMCheck)` in runLayer → "no
// oom_check in the docker report".
const oomCheckWire = `{"detail":"the engine reported the memory kill: OOMKilled=true and the oom event","exit_code":137,"image":"gitea.dooplex.hu/admin/felhom-controller:0.300.0","oom_event":true,"oom_killed":true,"result":"pass"}`
func TestDocker_OOMCheckReachesTheHubUnchanged(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}},
DockerEngine: "29.8.2", Authority: "ring0", OOMCheck: json.RawMessage(oomCheckWire)}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
found := false
for _, b := range h.bodies {
var m map[string]json.RawMessage
if err := json.Unmarshal(b, &m); err != nil {
t.Fatal(err)
}
var layer string
json.Unmarshal(m["layer"], &layer)
oc, has := m["oom_check"]
if layer != LayerDocker {
if has {
t.Fatalf("the %s report carries an oom_check: %s", layer, oc)
}
continue
}
found = true
if !has {
t.Fatalf("no oom_check in the docker report: %s", b)
}
if string(oc) != oomCheckWire {
t.Fatalf("oom_check changed on the way:\n got %s\nwant %s", oc, oomCheckWire)
}
}
if !found {
t.Fatalf("no docker report posted: %s", calls(w))
}
}
+1
View File
@@ -144,6 +144,7 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
rep.OOMCheck = rawOrNil(wr.OOMCheck)
wantEngine := ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
+21
View File
@@ -150,3 +150,24 @@ func must(t *testing.T, err error) {
t.Fatal(err)
}
}
// R-528: a kept docker report (the agent was killed) still carries the oom_check object to the hub unchanged.
// COMPANION RED-PROOF: drop `rep.OOMCheck = rawOrNil(wr.OOMCheck)` in reportFromKept → "oom_check lost".
func TestR868_KeptCopyCarriesTheOOMCheck(t *testing.T) {
w := &fakeWrapper{t: t}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
ring := 0
kept := WrapperReport{Mode: "apply", Layer: LayerDocker, RunID: "20261007T020000Z", Trigger: "night", Ring: &ring, VMID: 9201,
ReleaseID: "ring0-20261007T020000Z", HealthBefore: guestOK(), HealthAfter: guestOK(), DockerEngine: "29.8.2",
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, OOMCheck: json.RawMessage(oomCheckWire)}
b, _ := json.Marshal(kept)
must(t, os.WriteFile(reportFile(l.PlanDir, kept.RunID, LayerDocker, "apply"), b, 0o600))
if n := l.SendUnsent(context.Background()); n != 1 || len(h.bodies) != 1 {
t.Fatalf("sent %d, bodies %d", n, len(h.bodies))
}
var m map[string]json.RawMessage
must(t, json.Unmarshal(h.bodies[0], &m))
if string(m["oom_check"]) != oomCheckWire {
t.Fatalf("oom_check lost or changed: %q", m["oom_check"])
}
}