v0.145.0 code: the OS wrapper repairs dpkg's update journal by itself after a power cut (R-876); restore-test first check 30 min after start (R-874); neutral "sent late" text (R-875)
gates / gates (push) Successful in 20s
gates / gates (push) Successful in 20s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+30
-7
@@ -63,6 +63,9 @@ SNAPSHOT_LIST = "/etc/apt/sources.list.d/felhom-os-snapshot.list"
|
||||
APT_ENV = ["env", "DEBIAN_FRONTEND=noninteractive", "APT_LISTCHANGES_FRONTEND=none", "NEEDRESTART_MODE=l", "LC_ALL=C"]
|
||||
DPKG_OPTS = ["-o", "Dpkg::Options::=--force-confold", "-o", "Dpkg::Options::=--force-confdef"]
|
||||
MIN_FREE = 500 * 1024 * 1024
|
||||
# R-876: dpkg's state in ONE call — `--audit`, a marker line, then the update journal's file names.
|
||||
JOURNAL_MARK = "@@FELHOM-DPKG-JOURNAL@@"
|
||||
DPKG_STATE_SCRIPT = "dpkg --audit; echo " + JOURNAL_MARK + "; ls -A /var/lib/dpkg/updates 2>/dev/null; true"
|
||||
# The installer's ROOT-OWNED record (felhom-host-install.sh `state_set mode`); the agent cannot write it.
|
||||
INSTALL_STATE = "/var/lib/felhom-install/state.json"
|
||||
# Kernel, boot and firmware packages are the SLOW lane on the host whatever their origin (`11` C3, §5.2): a host
|
||||
@@ -902,20 +905,34 @@ class Apply:
|
||||
self.report["health_after"] = self.health()
|
||||
return 0
|
||||
|
||||
def repair(self):
|
||||
rc, before, _ = self.x(["dpkg", "--audit"])
|
||||
def dpkg_state(self):
|
||||
"""`dpkg --audit` AND dpkg's update journal, in ONE call (R-876, agent v0.145.0). A crash in the middle of an
|
||||
install can leave `/var/lib/dpkg/updates/` non-empty while `--audit` reads clean — measured on demo-hp
|
||||
2026-10-05 — and that journal is exactly what apt refuses on ("dpkg was interrupted"). One `sh -c` with a
|
||||
constant script keeps R-845's speed: a clean pass still costs one call here, as before."""
|
||||
rc, out, _ = self.x(["sh", "-c", DPKG_STATE_SCRIPT])
|
||||
audit, _, journal = out.partition(JOURNAL_MARK + "\n")
|
||||
return audit, [l for l in journal.split() if l]
|
||||
|
||||
def repair(self, force=False):
|
||||
before, journal = self.dpkg_state()
|
||||
configured = len([l for l in before.splitlines() if l.startswith(" ")])
|
||||
fixed = 0
|
||||
after = ""
|
||||
if before.strip(): # nothing half-done → nothing to run (R-845: two calls saved on every clean pass)
|
||||
after, journal_after = "", []
|
||||
# nothing half-done and no update journal → nothing to run (R-845: two calls saved on every clean pass).
|
||||
# R-876: the JOURNAL counts too, and `force` (apt said "dpkg was interrupted") always repairs.
|
||||
if before.strip() or journal or force:
|
||||
self.x(APT_ENV + ["dpkg", "--configure", "-a", "--force-confold"])
|
||||
rc2, out, err = self.x(APT_ENV + ["apt-get", "-f", "install", "-y", "-q"] + DPKG_OPTS)
|
||||
_, after, _ = self.x(["dpkg", "--audit"])
|
||||
after, journal_after = self.dpkg_state()
|
||||
fixed = len(re.findall(r"^Setting up ", out, re.M))
|
||||
self.report["repair"] = {"half_configured_before": configured, "fixed": fixed, "clean_after": after.strip() == ""}
|
||||
self.r.log(f"os-apply: REPAIR configured={configured} fixed={fixed}")
|
||||
self.report["repair"] = {"half_configured_before": configured, "journal_before": len(journal), "fixed": fixed,
|
||||
"clean_after": after.strip() == "" and not journal_after}
|
||||
self.r.log(f"os-apply: REPAIR configured={configured} journal={len(journal)} fixed={fixed}" + (" forced" if force else ""))
|
||||
if after.strip():
|
||||
raise Refused("R13", "dpkg is still broken after the repair: " + after.strip().splitlines()[0])
|
||||
if journal_after:
|
||||
raise Refused("R13", f"dpkg's update journal is still not empty after the repair ({len(journal_after)} file(s))")
|
||||
|
||||
def pending_fast(self):
|
||||
"""Ring 0 (select pending-fast): every pending upgrade of an INSTALLED package whose every origin is Debian /
|
||||
@@ -1032,6 +1049,12 @@ class Apply:
|
||||
raise Refused("R8", f"free space {free} B is below max(500 MB, 3 x download {need} B)")
|
||||
t0 = time.time()
|
||||
rc, out, err = self.x(APT_ENV + ["apt-get", "-y", "-q"] + DPKG_OPTS + args)
|
||||
if rc != 0 and "dpkg was interrupted" in (out + err):
|
||||
# R-876 (belt): apt says dpkg was interrupted although the repair found nothing — repair and
|
||||
# retry ONCE. Never a loop.
|
||||
self.r.log("os-apply: INTERRUPTED apt says dpkg was interrupted — repairing and retrying once")
|
||||
self.repair(force=True)
|
||||
rc, out, err = self.x(APT_ENV + ["apt-get", "-y", "-q"] + DPKG_OPTS + args)
|
||||
secs = time.time() - t0
|
||||
# dpkg says "Installing new version of config file X" when X was NOT changed locally (the package's new
|
||||
# version is taken), and "Configuration file 'X'" + "Keeping old config file" when it was (--force-confold
|
||||
|
||||
@@ -154,6 +154,8 @@ class Fake:
|
||||
if cmd == "dpkg" and a[1] == "--audit":
|
||||
return 0, self.dpkg_audit, ""
|
||||
if cmd == "dpkg" and a[1] == "--configure":
|
||||
self.configured_calls = getattr(self, "configured_calls", 0) + 1
|
||||
self.dpkg_journal = [] # `dpkg --configure -a` replays and empties the update journal
|
||||
return 0, "", ""
|
||||
if cmd == "fuser":
|
||||
return (0, " 123", "") if self.lock_held else (1, "", "")
|
||||
@@ -176,6 +178,9 @@ class Fake:
|
||||
if "--print-uris" in a or "-s" in a:
|
||||
return self.sim(a) # --print-uris prints and installs nothing, with or without -s (9202, 2026-10-05)
|
||||
if "install" in a:
|
||||
if getattr(self, "dpkg_journal", []):
|
||||
# real apt (demo-hp 2026-10-05): a non-empty update journal refuses every install
|
||||
return 100, "", "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem.\n"
|
||||
if self.install_rc:
|
||||
return self.install_rc, "", "E: boom"
|
||||
for x in a:
|
||||
@@ -220,6 +225,8 @@ class Fake:
|
||||
return 0, "", ""
|
||||
if cmd == "getent":
|
||||
return 0, "1.2.3.4 deb.debian.org\n", ""
|
||||
if cmd == "sh" and a[2] == osapply.DPKG_STATE_SCRIPT:
|
||||
return 0, self.dpkg_audit + osapply.JOURNAL_MARK + "\n" + "".join(j + "\n" for j in getattr(self, "dpkg_journal", [])), ""
|
||||
if cmd == "sh":
|
||||
if "vmlinuz" in a[2]:
|
||||
return 0, "/boot/vmlinuz-7.0.2-6-pve\n/boot/vmlinuz-7.0.14-20-pve\n", ""
|
||||
@@ -1123,3 +1130,52 @@ class AgentDiesMidPass(unittest.TestCase):
|
||||
self.assertEqual(len(f.saved_reports), 1, "the kept copy is the only way this report reaches the hub")
|
||||
self.assertEqual(len(f.saved_reports[0][1]["upgraded"]), 2)
|
||||
self.assertTrue(any(l.startswith("os-apply: DONE") for l in journal), "the journal must still get the DONE line")
|
||||
|
||||
|
||||
class CrashLeftTheJournal(unittest.TestCase):
|
||||
"""R-876 — THE MEASURED SHAPE (demo-hp 2026-10-05, a crash while dpkg unpacked): `dpkg --audit` CLEAN, but
|
||||
/var/lib/dpkg/updates holds 3 files; apt refuses every install ("dpkg was interrupted"). v0.144.1 logged
|
||||
`REPAIR configured=0 fixed=0`, then `FAILED rc=100`, every pass, until a person ran `dpkg --configure -a`.
|
||||
COMPANION RED-PROOFS: drop `or journal` from repair()'s condition -> test 1 fails; drop the interrupted-retry
|
||||
-> test 3 fails; make repair() always run -> test 2 fails (R-845's speed)."""
|
||||
|
||||
def test_the_next_pass_repairs_by_itself_and_finishes(self):
|
||||
f = Fake()
|
||||
f.dpkg_audit = ""
|
||||
f.dpkg_journal = ["0000", "0001", "0002"]
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(rep["repair"]["journal_before"], 3)
|
||||
self.assertTrue(rep["repair"]["clean_after"])
|
||||
self.assertEqual(len(rep["upgraded"]), 2)
|
||||
cfg = next(i for i, c in enumerate(f.calls) if c[0] == "guest" and "--configure" in c[2])
|
||||
inst = next(i for i, c in enumerate(f.calls) if c[0] == "guest" and "install" in c[2] and "-s" not in c[2]
|
||||
and "-f" not in c[2] and "--print-uris" not in c[2])
|
||||
self.assertLess(cfg, inst, "the repair must run before the install")
|
||||
self.assertFalse(any("INTERRUPTED" in l for l in f.logs), "the journal check must catch it BEFORE apt refuses")
|
||||
|
||||
def test_a_clean_pass_still_costs_one_state_call(self):
|
||||
f = Fake()
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
state = [c for c in f.calls if c[0] == "guest" and c[2][:2] == ["sh", "-c"] and c[2][2] == osapply.DPKG_STATE_SCRIPT]
|
||||
self.assertEqual(len(state), 1, "R-845: a clean pass reads dpkg's state ONCE and runs no repair")
|
||||
self.assertEqual(getattr(f, "configured_calls", 0), 0)
|
||||
|
||||
def test_apt_interrupted_is_repaired_and_retried_once(self):
|
||||
"""The belt: the journal probe saw nothing (a race, an odd layout), apt still says interrupted."""
|
||||
f = Fake()
|
||||
orig = f.emulate
|
||||
state = {"first": True}
|
||||
|
||||
def emulate(argv):
|
||||
a = [x for x in argv if not re.match(r"^[A-Z_]+=", x) and x != "env"]
|
||||
if a[0] == "apt-get" and "install" in a and "-s" not in a and "-f" not in a and "--print-uris" not in a and state["first"]:
|
||||
state["first"] = False
|
||||
return 100, "", "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem.\n"
|
||||
return orig(argv)
|
||||
f.emulate = emulate
|
||||
rc, rep = run(f)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(any("INTERRUPTED" in l for l in f.logs), f.logs)
|
||||
self.assertTrue(any(l.startswith("os-apply: REPAIR ") and l.endswith("forced") for l in f.logs), f.logs)
|
||||
|
||||
Reference in New Issue
Block a user