R-528 (09 decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill
felhom-os-apply: a docker-layer apply runs oom_check() after health_after and reports
"oom_check": {result pass|fail|error, oom_killed, oom_event, exit_code, image, detail}.
One throwaway container (the controller's image, --pull never, --network none, 64m cap,
label felhom.oomcheck=1) runs dd bs=200M; pass only with OOMKilled=true AND the oom event
(read after a 2 s settle, --until = guest epoch + 1: measured on demo-hp, an --until taken
right after the run missed the event). docker rm -f always runs in a finally; every call
is bounded (<= 90 s). It never changes the step's outcome or health. New wrapper-only mode
"oom-check" (docker layer) runs the check alone; check_guest etc. still apply.
Agent: WrapperReport/Report gain OOMCheck (json:"oom_check"), copied unchanged in runLayer
and in the kept-copy path.
Tests: 9 wrapper tests + 2 Go tests, each red-proofed (audits/readback-2026-10-07/F/red-*.txt).
Also: test_felhom_os_apply.py's `if __name__` sat mid-file, so 11 tests (UnsentReport,
SaveReportOnDisk, AgentDiesMidPass, CrashLeftTheJournal) never ran as a script or from
TestWrapperSuite; moved to the end (they pass).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+104
-1
@@ -32,6 +32,14 @@
|
||||
# held packages, kernel taint, the crash guard; guest Debian, Docker engine, containerd, live-restore.
|
||||
# live-restore-on (v0.142.0, layer guest) the ONE-TIME step of `09` decision 87: merge `"live-restore": true`
|
||||
# into the guest's /etc/docker/daemon.json and `systemctl reload docker`. NEVER a restart (R-835).
|
||||
# oom-check (R-528, `09` decision 157; layer docker, lane slow) ONLY the memory-kill check below: no apt, no engine
|
||||
# change, no authority needed. Wrapper-only — the agent never writes this plan; run it by hand as root.
|
||||
# R-528 (`09` decision 157): a docker-layer APPLY also runs `oom_check()` after health_after and reports it as
|
||||
# "oom_check": {"result": "pass"|"fail"|"error", "oom_killed": bool, "oom_event": bool, "exit_code": int|null,
|
||||
# "image": str|null, "detail": str}
|
||||
# — one throwaway container (the controller's own image, no network / volume / port, 64 MB cap) is made to exceed its
|
||||
# memory; "pass" only when the engine says OOMKilled=true AND emits the `oom` event. It never changes the step's
|
||||
# outcome or health: the hub decides whether the engine set can be approved. Pinned by the OOMCheck tests.
|
||||
# Output: log lines on stderr and the journal (tag felhom-os-apply); the LAST stdout line is
|
||||
# OSAPPLY-REPORT <one JSON object>
|
||||
# which is what the agent parses. Exit 0 = done; 2 = refused (nothing changed); 3 = failed during install.
|
||||
@@ -68,6 +76,16 @@ JOURNAL_MARK = "@@FELHOM-DPKG-JOURNAL@@"
|
||||
DPKG_STATE_SCRIPT = "dpkg --audit; echo " + JOURNAL_MARK + "; ls -A /var/lib/dpkg/updates 2>/dev/null; true"
|
||||
# The installer's ROOT-OWNED record (felhom-host-install.sh `state_set mode`); the agent cannot write it.
|
||||
INSTALL_STATE = "/var/lib/felhom-install/state.json"
|
||||
# R-528: the memory-kill check (oom_check). One 200 MB block under a 64 MB cap: measured on Docker 29.8.2 to be
|
||||
# OOM-killed with OOMKilled=true and an `oom` event. Every call is bounded: timeouts (the clock read twice) + the
|
||||
# settle wait stay within 90 s.
|
||||
OOMCHECK_PREFIX = "felhom-oomcheck-"
|
||||
OOMCHECK_SCRIPT = "dd if=/dev/zero of=/dev/null bs=200M count=1"
|
||||
OOMCHECK_TIMEOUTS = {"image": 10, "clock": 5, "run": 30, "inspect": 10, "events": 10, "rm": 15}
|
||||
# Measured on demo-hp 9201 (Docker 29.8.2, 2026-10-06, audits/readback-2026-10-07/F/F1, F2): an `--until` taken right
|
||||
# after the run MISSED the oom event although OOMKilled=true; after a 2 s wait and `--until` = guest epoch + 1 it is
|
||||
# seen. Pinned by test_events_window_ends_after_the_settle_wait.
|
||||
OOMCHECK_SETTLE = 2
|
||||
# Kernel, boot and firmware packages are the SLOW lane on the host whatever their origin (`11` C3, §5.2): a host
|
||||
# reboot is needed for them to take effect, and a bad one can stop the box from booting.
|
||||
HOST_SLOW_RE = re.compile(r"^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|"
|
||||
@@ -405,7 +423,7 @@ class Apply:
|
||||
|
||||
def check_plan(self, plan):
|
||||
mode = plan.get("mode", "apply")
|
||||
if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update"):
|
||||
if mode not in ("apply", "inventory", "health", "facts", "live-restore-on", "bundle", "agent_update", "oom-check"):
|
||||
raise Refused("R11", f"unknown mode {mode!r}")
|
||||
if mode == "agent_update":
|
||||
if plan.get("layer") != "host":
|
||||
@@ -427,6 +445,8 @@ class Apply:
|
||||
raise Refused("R11", "facts is a host-layer mode (it reads the host and the guest)")
|
||||
if mode == "live-restore-on" and layer != "guest":
|
||||
raise Refused("R11", "live-restore-on is a guest-layer mode")
|
||||
if mode == "oom-check" and layer != "docker":
|
||||
raise Refused("R11", "oom-check is a docker-layer mode (it checks the guest's Docker engine)")
|
||||
if plan.get("undo") and layer != "docker":
|
||||
raise Refused("R5", "an undo (downgrade) exists only for the Docker layer, inside a signed job")
|
||||
vmid = plan.get("vmid")
|
||||
@@ -951,6 +971,10 @@ class Apply:
|
||||
log = self.r.log
|
||||
if self.mode == "live-restore-on":
|
||||
return self.live_restore_on()
|
||||
if self.mode == "oom-check":
|
||||
# R-528: the check alone — no apt, no engine change; check_guest above still applies.
|
||||
self.report["oom_check"] = self.oom_check()
|
||||
return 0
|
||||
self.who, self.allow_downgrade = ("fast", False)
|
||||
if self.layer == "docker" and self.mode == "apply":
|
||||
self.who, self.allow_downgrade = self.docker_authority(plan)
|
||||
@@ -990,8 +1014,87 @@ class Apply:
|
||||
self.report["docker_engine"] = out_v.strip() if rc_v == 0 and out_v.strip() else "unknown"
|
||||
self.report["reboot_scanned"] = "reboot_needed" in self.report
|
||||
self.report["health_after"] = self.health()
|
||||
if self.layer == "docker" and self.mode == "apply":
|
||||
# R-528 (`09` decision 157): does the engine report a memory kill? Reported only — never the outcome.
|
||||
self.report["oom_check"] = self.oom_check()
|
||||
return 0
|
||||
|
||||
# ---------- R-528: the memory-kill check ----------
|
||||
def oom_check(self):
|
||||
"""Run one throwaway container over its memory cap in the guest and read what the engine says about it.
|
||||
Never raises: any failure becomes result "error". The container is ALWAYS removed (finally), and a failed
|
||||
removal is named in the detail."""
|
||||
t = OOMCHECK_TIMEOUTS
|
||||
res = {"result": "error", "oom_killed": False, "oom_event": False, "exit_code": None, "image": None, "detail": ""}
|
||||
log = self.r.log
|
||||
try:
|
||||
rc, out, err = self.g(["docker", "inspect", "-f", "{{.Config.Image}}", "felhom-controller"], timeout=t["image"])
|
||||
except Exception as e:
|
||||
rc, out, err = -1, "", str(e)
|
||||
img = out.strip() if rc == 0 else ""
|
||||
if not img or any(c.isspace() for c in img):
|
||||
res["detail"] = f"the controller's image could not be read (rc={rc}): {(err or out).strip()[:200]}"
|
||||
log(f"os-apply: OOM-CHECK error — {res['detail']}")
|
||||
return res
|
||||
res["image"] = img
|
||||
name = f"{OOMCHECK_PREFIX}{os.getpid()}-{os.urandom(4).hex()}"
|
||||
notes = []
|
||||
try:
|
||||
t0 = self.guest_epoch()
|
||||
if t0 is None:
|
||||
raise RuntimeError("the guest clock could not be read")
|
||||
rrc, rout, rerr = self.g(["docker", "run", "--name", name, "--pull", "never", "--network", "none",
|
||||
"--memory", "64m", "--memory-swap", "64m", "--label", "felhom.oomcheck=1",
|
||||
"--entrypoint", "sh", img, "-c", OOMCHECK_SCRIPT], timeout=t["run"])
|
||||
self.r.sleep(OOMCHECK_SETTLE) # the engine publishes the oom event a moment after the run returns (F1/F2)
|
||||
t1 = self.guest_epoch()
|
||||
if t1 is None:
|
||||
t1 = t0 + t["run"] + OOMCHECK_SETTLE + 1
|
||||
irc, iout, ierr = self.g(["docker", "inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}", name], timeout=t["inspect"])
|
||||
if irc != 0:
|
||||
raise RuntimeError(f"the check container could not be inspected (run rc={rrc}: {(rerr or rout).strip()[:120]}; "
|
||||
f"inspect rc={irc}: {(ierr or iout).strip()[:120]})")
|
||||
f = iout.split()
|
||||
res["oom_killed"] = bool(f) and f[0] == "true"
|
||||
try:
|
||||
res["exit_code"] = int(f[1]) if len(f) > 1 else None
|
||||
except ValueError:
|
||||
res["exit_code"] = None
|
||||
erc, eout, eerr = self.g(["docker", "events", "--since", str(t0 - 1), "--until", str(t1 + 1),
|
||||
"--filter", f"container={name}", "--filter", "event=oom",
|
||||
"--format", "{{.Action}}"], timeout=t["events"])
|
||||
if erc != 0:
|
||||
notes.append(f"the event read failed (rc={erc}): {(eerr or eout).strip()[:120]}")
|
||||
res["oom_event"] = erc == 0 and any(l.strip() == "oom" for l in eout.splitlines())
|
||||
if res["oom_killed"] and res["oom_event"]:
|
||||
res["result"] = "pass"
|
||||
notes.insert(0, "the engine reported the memory kill: OOMKilled=true and the oom event")
|
||||
else:
|
||||
res["result"] = "fail"
|
||||
miss = [w for w, ok in (("OOMKilled=true", res["oom_killed"]), ("the oom event", res["oom_event"])) if not ok]
|
||||
notes.insert(0, f"the engine did not report the memory kill: missing {' and '.join(miss)} (exit code {res['exit_code']})")
|
||||
except Exception as e:
|
||||
res["result"] = "error"
|
||||
notes.insert(0, f"the check could not finish: {type(e).__name__}: {str(e)[:200]}")
|
||||
finally:
|
||||
try:
|
||||
mrc, mout, merr = self.g(["docker", "rm", "-f", name], timeout=t["rm"])
|
||||
if mrc != 0 and "no such container" not in (merr + mout).lower():
|
||||
notes.append(f"the check container {name} could not be removed (rc={mrc}): {(merr or mout).strip()[:120]}")
|
||||
except Exception as e:
|
||||
notes.append(f"the check container {name} could not be removed: {type(e).__name__}: {str(e)[:120]}")
|
||||
res["detail"] = "; ".join(notes)
|
||||
log(f"os-apply: OOM-CHECK result={res['result']} oom_killed={res['oom_killed']} oom_event={res['oom_event']} "
|
||||
f"exit={res['exit_code']} image={img} — {res['detail']}")
|
||||
return res
|
||||
|
||||
def guest_epoch(self):
|
||||
rc, out, _ = self.g(["date", "+%s"], timeout=OOMCHECK_TIMEOUTS["clock"])
|
||||
try:
|
||||
return int(out.strip()) if rc == 0 else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def dpkg_state(self):
|
||||
"""`dpkg --audit` AND dpkg's update journal, in ONE call (R-876, agent v0.145.0). A crash in the middle of an
|
||||
install can leave `/var/lib/dpkg/updates/` non-empty while `--audit` reads clean — measured on demo-hp
|
||||
|
||||
@@ -83,7 +83,7 @@ class Fake:
|
||||
return self.clock
|
||||
|
||||
def sleep(self, s):
|
||||
pass
|
||||
self.sleeps = getattr(self, "sleeps", []) + [(len(self.calls), s)] # (calls made before it, seconds)
|
||||
|
||||
def verify_sig(self, signers, key_id, ns, blob, sig):
|
||||
self.verified = (signers, key_id, ns, blob, sig)
|
||||
@@ -196,6 +196,27 @@ class Fake:
|
||||
return 0, self.engine + "\n", ""
|
||||
if cmd == "docker" and a[1:3] == ["ps", "-q"]:
|
||||
return 0, "".join(i + "\n" for i in self.ids), ""
|
||||
# R-528: the memory-kill check's engine (oom_image None = unreadable; oom_state / oom_event the engine's answer)
|
||||
if cmd == "date" and a[1:] == ["+%s"]:
|
||||
# each read is 3 s later than the last, so the order of the reads is visible in the values
|
||||
self.oom_epochs = getattr(self, "oom_epochs", []) + [int(self.clock) + 3 * len(getattr(self, "oom_epochs", []))]
|
||||
return 0, f"{self.oom_epochs[-1]}\n", ""
|
||||
if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.Config.Image}}"]:
|
||||
img = getattr(self, "oom_image", "gitea.dooplex.hu/admin/felhom-controller:0.300.0")
|
||||
return (0, img + "\n", "") if img is not None else (1, "", "Error: No such object: felhom-controller")
|
||||
if cmd == "docker" and a[1] == "run":
|
||||
self.oom_runs = getattr(self, "oom_runs", []) + [a]
|
||||
return 137, "", ""
|
||||
if cmd == "docker" and a[1:4] == ["inspect", "-f", "{{.State.OOMKilled}} {{.State.ExitCode}}"]:
|
||||
if getattr(self, "oom_inspect_raises", False):
|
||||
raise subprocess.TimeoutExpired(a, 10)
|
||||
return 0, getattr(self, "oom_state", "true 137") + "\n", ""
|
||||
if cmd == "docker" and a[1] == "events":
|
||||
self.oom_events_argv = a
|
||||
return 0, ("oom\n" if getattr(self, "oom_event", True) else ""), ""
|
||||
if cmd == "docker" and a[1:3] == ["rm", "-f"]:
|
||||
self.oom_removed = getattr(self, "oom_removed", []) + a[3:]
|
||||
return 0, a[3] + "\n", ""
|
||||
if cmd == "docker" and a[1] == "inspect":
|
||||
mounts = {"aaa111": "/felhom-controller|/var/run/docker.sock;/app/data;", "bbb222": "/app|/data;"}
|
||||
return 0, mounts.get(a[-1], "/other|;") + "\n", ""
|
||||
@@ -997,8 +1018,6 @@ class RealSignatureCheck(unittest.TestCase):
|
||||
self.assertNotEqual(r.verify_sig(self.signers, "someone-else", "felhom-op-v1", blob, sig), 0)
|
||||
self.assertNotEqual(r.verify_sig(self.signers, "felhom-op-1", "felhom-op-v1", blob, self.sign(blob, ns="other-ns")), 0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class UnsentReport(unittest.TestCase):
|
||||
@@ -1179,3 +1198,131 @@ class CrashLeftTheJournal(unittest.TestCase):
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(any("INTERRUPTED" in l for l in f.logs), f.logs)
|
||||
self.assertTrue(any(l.startswith("os-apply: REPAIR ") and l.endswith("forced") for l in f.logs), f.logs)
|
||||
|
||||
|
||||
class OOMCheck(unittest.TestCase):
|
||||
"""R-528 (`09` decision 157): after a Docker engine step the wrapper proves the engine reports a memory kill
|
||||
(OOMKilled=true AND the `oom` event). Reported only; the hub decides. Red-proofs: audits/readback-2026-10-07/F/."""
|
||||
|
||||
def apply(self, **kw):
|
||||
f = docker_fake(signed=signed_job())
|
||||
for k, v in kw.items():
|
||||
setattr(f, k, v)
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
return f, rep
|
||||
|
||||
def test_pass_when_oomkilled_and_the_event(self):
|
||||
f, rep = self.apply()
|
||||
oc = rep["oom_check"]
|
||||
self.assertEqual(oc["result"], "pass", oc)
|
||||
self.assertEqual((oc["oom_killed"], oc["oom_event"], oc["exit_code"]), (True, True, 137))
|
||||
self.assertEqual(oc["image"], "gitea.dooplex.hu/admin/felhom-controller:0.300.0")
|
||||
self.assertEqual(sorted(oc), ["detail", "exit_code", "image", "oom_event", "oom_killed", "result"])
|
||||
run_argv = f.oom_runs[0]
|
||||
name = run_argv[run_argv.index("--name") + 1]
|
||||
self.assertTrue(re.match(r"^felhom-oomcheck-[0-9]+-[0-9a-f]{8}$", name), name)
|
||||
for flag, val in (("--pull", "never"), ("--network", "none"), ("--memory", "64m"), ("--memory-swap", "64m"),
|
||||
("--label", "felhom.oomcheck=1"), ("--entrypoint", "sh")):
|
||||
self.assertEqual(run_argv[run_argv.index(flag) + 1], val, flag)
|
||||
self.assertNotIn("-v", run_argv)
|
||||
self.assertNotIn("-p", run_argv)
|
||||
self.assertEqual(run_argv[-3:], ["gitea.dooplex.hu/admin/felhom-controller:0.300.0", "-c", osapply.OOMCHECK_SCRIPT])
|
||||
self.assertIn(f"container={name}", f.oom_events_argv)
|
||||
self.assertIn("event=oom", f.oom_events_argv)
|
||||
self.assertEqual(f.oom_removed, [name], "the check container must be removed")
|
||||
self.assertTrue(rep["health_after"], "health is read before the check")
|
||||
# bounded: every call has a timeout and the clock is read twice — the worst case stays within 90 s
|
||||
self.assertLessEqual(sum(osapply.OOMCHECK_TIMEOUTS.values()) + osapply.OOMCHECK_TIMEOUTS["clock"]
|
||||
+ osapply.OOMCHECK_SETTLE, 90)
|
||||
|
||||
def test_events_window_ends_after_the_settle_wait(self):
|
||||
# Measured (F1/F2): an --until taken right after the run missed the oom event. The window must end after a
|
||||
# wait of >= 2 s that comes AFTER the run, at the guest epoch read after that wait, + 1.
|
||||
f, rep = self.apply()
|
||||
run_i = next(i for i, c in enumerate(f.calls) if c[-1][:2] == ["docker", "run"])
|
||||
date_i = [i for i, c in enumerate(f.calls) if c[-1] == ["date", "+%s"]]
|
||||
waits = [(i, s) for i, s in getattr(f, "sleeps", []) if i > run_i]
|
||||
self.assertTrue(waits and waits[0][1] >= 2, f"no settle wait after the run: {getattr(f, 'sleeps', None)}")
|
||||
self.assertTrue(date_i[-1] >= waits[0][0], "the end epoch must be read after the wait")
|
||||
ev = f.oom_events_argv
|
||||
since, until = int(ev[ev.index("--since") + 1]), int(ev[ev.index("--until") + 1])
|
||||
self.assertEqual(until, f.oom_epochs[-1] + 1, "until = the guest epoch read after the wait, + 1")
|
||||
self.assertGreater(until, f.oom_epochs[0] + 1)
|
||||
self.assertLess(since, f.oom_epochs[0] + 1)
|
||||
|
||||
def test_oomkilled_false_is_fail(self):
|
||||
f, rep = self.apply(oom_state="false 0")
|
||||
oc = rep["oom_check"]
|
||||
self.assertEqual(oc["result"], "fail", oc)
|
||||
self.assertIn("OOMKilled=true", oc["detail"])
|
||||
self.assertTrue(oc["oom_event"])
|
||||
self.assertEqual(len(f.oom_removed), 1)
|
||||
|
||||
def test_no_event_is_fail(self):
|
||||
f, rep = self.apply(oom_event=False)
|
||||
oc = rep["oom_check"]
|
||||
self.assertEqual(oc["result"], "fail", oc)
|
||||
self.assertIn("the oom event", oc["detail"])
|
||||
self.assertTrue(oc["oom_killed"])
|
||||
|
||||
def test_image_unreadable_is_error(self):
|
||||
f, rep = self.apply(oom_image=None)
|
||||
oc = rep["oom_check"]
|
||||
self.assertEqual(oc["result"], "error", oc)
|
||||
self.assertIsNone(oc["image"])
|
||||
self.assertIn("image could not be read", oc["detail"])
|
||||
self.assertFalse(hasattr(f, "oom_runs"), "no container is started without an image")
|
||||
|
||||
def test_container_removed_even_when_inspect_raises(self):
|
||||
f, rep = self.apply(oom_inspect_raises=True)
|
||||
oc = rep["oom_check"]
|
||||
self.assertEqual(oc["result"], "error", oc)
|
||||
self.assertEqual(len(f.oom_removed), 1, "the container must be removed even when inspect raised")
|
||||
self.assertTrue(f.oom_removed[0].startswith(osapply.OOMCHECK_PREFIX))
|
||||
self.assertNotIn("failed", rep, "the check never turns the step into a failure")
|
||||
|
||||
def test_never_on_guest_or_host_or_in_health_mode(self):
|
||||
g = Fake()
|
||||
rc, rep = run(g)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
h = Fake()
|
||||
h.plan["layer"] = "host"
|
||||
h.plan["packages"] = [{"name": "bash", "version": "5.2.37-2+b10", "origin": "Debian"}]
|
||||
h.installed["bash"] = "5.2.37-2+b9"
|
||||
h.live["bash"] = {"5.2.37-2+b10"}
|
||||
rc_h, rep_h = run(h)
|
||||
self.assertEqual(rc_h, 0, rep_h)
|
||||
d = docker_fake(signed=signed_job())
|
||||
d.plan["mode"] = "health"
|
||||
rc_d, rep_d = run(d)
|
||||
self.assertEqual(rc_d, 0, rep_d)
|
||||
for f, r in ((g, rep), (h, rep_h), (d, rep_d)):
|
||||
self.assertNotIn("oom_check", r)
|
||||
self.assertFalse(hasattr(f, "oom_runs"), r.get("layer"))
|
||||
self.assertFalse(any(c[-1][:2] == ["docker", "run"] for c in f.calls))
|
||||
|
||||
def test_mode_oom_check_runs_only_the_check(self):
|
||||
f = docker_fake() # no authority: the check changes nothing, so it needs none
|
||||
f.plan["mode"], f.plan["packages"] = "oom-check", []
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(rep["oom_check"]["result"], "pass", rep)
|
||||
self.assertFalse(any("apt-get" in c[-1] or "dpkg-query" in c[-1] for c in f.calls), f.calls)
|
||||
self.assertEqual(len(f.oom_removed), 1)
|
||||
|
||||
def test_mode_oom_check_keeps_the_refusals(self):
|
||||
f = docker_fake()
|
||||
f.plan["mode"], f.plan["packages"] = "oom-check", []
|
||||
f.files["/etc/pve/lxc/9201.conf"] = "arch: amd64\n"
|
||||
rc, rep = run(f)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R10"), rep)
|
||||
self.assertFalse(hasattr(f, "oom_runs"))
|
||||
g = Fake()
|
||||
g.plan["mode"] = "oom-check"
|
||||
rc, rep = run(g)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R11"), rep)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user