diff --git a/CHANGELOG.md b/CHANGELOG.md index 81484ac..ef1de0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,11 @@ Commit only. Each part can be released alone; none needs release 2b. No `configs copy), and its root step has a 90-second deadline (it runs on the start path and inside the report loop). (3) **The CA copy keeps its previous certificate behind the new one** (`configs/felhom-os-apply`, mode `pve-ca-copy`): the copy was replaced before anything could be proven with the new root, and Proxmox serves its old certificate until - pveproxy is reloaded — an agent that restarted in that window had no root that verified it. At most two - certificates; the same source is still „same”. `configs/` changes again, so the release needs both signed jobs. + pveproxy is reloaded — an agent that restarted in that window had no root that verified it. **Only for the + changeover:** at every later run the step makes one handshake to the local Proxmox with the NEW root alone, and + drops the previous root as soon as that verifies, or when the previous root is no longer a CA that has not ended. + The copy is compared exactly, so nothing the step did not check stays in it. To drop the old root at once by hand + (a root changed because it leaked): delete the copy. `configs/` changes again, so the release needs both signed jobs. (4) The profile-refresh stanza is built from the wrapper's ANSWER (its record of the last refresh comes back at every ask); the record file is a bonus — its folder is root-only on a box where a signed job made it first (read in source, not on a box). `rewritten` is absent when not known. (5) A watchdog give-up is forgotten when the guest is diff --git a/configs/felhom-os-apply b/configs/felhom-os-apply index ea00a43..a314f46 100755 --- a/configs/felhom-os-apply +++ b/configs/felhom-os-apply @@ -167,6 +167,7 @@ KERNEL_ACTIVE = ("staged", "oneshot", "judging", "reverting") PVE_CA_SOURCE = "/etc/pve/pve-root-ca.pem" PVE_CA_COPY = "/etc/felhom-agent/pve-root-ca.pem" # 0644 root: a CA certificate is public; the agent only reads it PVE_CA_MAX_BYTES = 64 * 1024 +PVE_LOCAL_API = "127.0.0.1:8006" # pveproxy on this host (the installer's proof uses the same) PEM_CERT_RE = re.compile(rb"-----BEGIN CERTIFICATE-----\s[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----") # The security-profile refresh (R-1388, R-1301, `11` §5.12; measured on demo-hp 2026-11-14, # felhom.eu audits/profile-refresh-spike-2026-11-14/). AppArmor keeps one cache folder per kernel feature set; a @@ -1213,11 +1214,30 @@ class Apply: return 0 # ---------- trust in Proxmox by a copy of its root CA (R-1380, D80) ---------- + def is_live_ca(self, block): + """One PEM certificate block: is it a CA certificate that has not ended? (openssl, the bytes on stdin).""" + try: + rc, out, _ = self.r.host(["openssl", "x509", "-noout", "-ext", "basicConstraints", "-checkend", "0"], 30, + stdin=block.decode("ascii") + "\n") + except (UnicodeDecodeError, subprocess.SubprocessError, OSError): + return False + return rc == 0 and re.search(r"\bCA:TRUE\b", out) is not None + + def pve_serves_with(self, cafile): + """Does the local Proxmox API serve a certificate that verifies with this CA file ALONE? One handshake to + PVE_LOCAL_API; nothing is sent. Any doubt (no answer, a timeout) is "no" — the previous root then stays.""" + try: + rc, out, err = self.r.host(["openssl", "s_client", "-connect", PVE_LOCAL_API, "-CAfile", cafile, "-no-CApath", + "-verify_return_error"], 20, stdin="") + except (subprocess.SubprocessError, OSError): + return False + return rc == 0 and "Verify return code: 0 (ok)" in (out + err) + def pve_ca_copy(self): """Copy exactly PVE_CA_SOURCE to exactly PVE_CA_COPY — only if the source is ONE CA certificate that has not ended — written whole or not at all (a root-owned temp file beside the target, fsync, rename). "same" when the - copy already starts with the source's bytes. The bytes that are checked are the bytes that are written first; - behind them the copy keeps its own previous certificate (see below), so the copy holds one or two.""" + copy is exactly the source's bytes. The bytes that are checked are the bytes that are written first; only + during a change of the root the copy also holds its previous certificate behind them (see below).""" r = self.r res = {"outcome": None, "path": PVE_CA_COPY} self.report["pve_ca_copy"] = res @@ -1254,18 +1274,30 @@ class Apply: have = r.read_bytes(PVE_CA_COPY) except OSError: have = None - if have is not None and have.startswith(data): - res["outcome"] = "same" # the source's certificate is already first in the copy + if have == data: + res["outcome"] = "same" + return 0 + # The copy the agent trusts by is replaced BEFORE anything can be proven with the new root (Proxmox serves its + # old certificate until pveproxy is reloaded). So at a CHANGE the copy's previous first certificate — written + # by this same step — is kept behind the new one, and the agent trusts both roots across the change, also + # after a restart. That second root is kept ONLY for the changeover: it is dropped as soon as Proxmox serves a + # certificate that verifies with the new root alone, and at once when it is no longer a CA that has not ended. + # The copy is compared EXACTLY: nothing this step did not check is ever left in it. (Review, 2026-11-14.) + # To drop the old root at once by hand (a root that was changed because it leaked): delete the copy. + prev = PEM_CERT_RE.findall(have) if have and b"PRIVATE KEY" not in have else [] + if len(prev) == 2 and have == data + prev[1] + b"\n": + over = self.pve_serves_with(PVE_CA_SOURCE) + if not over and self.is_live_ca(prev[1]): + res["outcome"], res["kept_previous"] = "same", True # the changeover is still on + return 0 + r.put_file(PVE_CA_COPY, data, 0o644) + res["outcome"], res["replaced"], res["kept_previous"], res["dropped_previous"] = "written", True, False, True + r.log(f"os-apply: PVE CA COPY the previous root is dropped from {PVE_CA_COPY} " + f"({'Proxmox serves a certificate of the new root' if over else 'the previous root is no longer a live CA'})") return 0 - # The copy the agent trusts by is replaced BEFORE anything can be proven with the new one (Proxmox serves its - # old certificate until pveproxy is reloaded). So the copy's previous FIRST certificate — written by this same - # step, so already checked as a CA — is kept behind the new one: the agent then trusts both roots across the - # change, also after a restart. At most two certificates; an ended one verifies nothing. (Review, 2026-11-14.) want, kept = data, False - if have: - prev = PEM_CERT_RE.findall(have) - if prev and b"PRIVATE KEY" not in have and prev[0].strip() != certs[0].strip(): - want, kept = data + prev[0] + b"\n", True + if prev and prev[0].strip() != certs[0].strip() and self.is_live_ca(prev[0]): + want, kept = data + prev[0] + b"\n", True r.put_file(PVE_CA_COPY, want, 0o644) res["outcome"], res["replaced"], res["kept_previous"] = "written", have is not None, kept r.log(f"os-apply: PVE CA COPY written {PVE_CA_COPY} sha256={res['sha256'][:16]} ends={res['not_after']} " diff --git a/configs/test_felhom_os_apply.py b/configs/test_felhom_os_apply.py index 46138a7..68bf0f0 100644 --- a/configs/test_felhom_os_apply.py +++ b/configs/test_felhom_os_apply.py @@ -2648,6 +2648,9 @@ class TFake(Fake): if copy is not None: self.tree[osapply.PVE_CA_COPY] = copy self.openssl_stdin, self.puts = [], [] + self.serves_new = False # does pveproxy serve a certificate that verifies with the NEW root alone? + self.dead_ca = set() # certificate texts openssl says are not a live CA (ended) + self.handshakes = [] self.plan = {"release_id": "pve-ca", "layer": "host", "mode": "pve-ca-copy"} def read_bytes(self, p): @@ -2664,6 +2667,12 @@ class TFake(Fake): self.tree[path] = data def host(self, argv, timeout=600, stdin=None): + if argv[0] == "openssl" and argv[1] == "s_client": + self.handshakes.append(argv) + return (0, "Verify return code: 0 (ok)\n", "") if self.serves_new else (1, "", "Verify return code: 19 (self-signed certificate in certificate chain)\n") + if argv[0] == "openssl" and "-checkend" in argv: + return (1, "X509v3 Basic Constraints: critical\n CA:TRUE\nCertificate will expire\n", "") if stdin.strip() in self.dead_ca else \ + (0, "X509v3 Basic Constraints: critical\n CA:TRUE\nCertificate will not expire\n", "") if argv[0] == "openssl": self.calls.append(("host", argv)) self.openssl_stdin.append(stdin) @@ -2704,26 +2713,62 @@ class PVECACopy(unittest.TestCase): self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], rep["pve_ca_copy"]["replaced"]), (0, "written", True), rep) # the new certificate FIRST, and the copy's previous one kept behind it: Proxmox serves its old certificate until # pveproxy is reloaded, and an agent that restarts in that window must still have a root that verifies it - self.assertTrue(f.tree[osapply.PVE_CA_COPY].startswith(CA_PEM_2)) - self.assertEqual(osapply.PEM_CERT_RE.findall(f.tree[osapply.PVE_CA_COPY]), - osapply.PEM_CERT_RE.findall(CA_PEM_2) + osapply.PEM_CERT_RE.findall(CA_PEM)) + bundle = f.tree[osapply.PVE_CA_COPY] + self.assertEqual(bundle, CA_PEM_2 + osapply.PEM_CERT_RE.findall(CA_PEM)[0] + b"\n") self.assertTrue(rep["pve_ca_copy"]["kept_previous"]) - self.assertEqual(f.openssl_stdin, [CA_PEM_2.decode()], "only the SOURCE is checked by openssl") - # the same source again: nothing is written (the copy already starts with it) - f2 = TFake(source=CA_PEM_2, copy=f.tree[osapply.PVE_CA_COPY]) - rc, rep = run(f2) - self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], f2.puts), (0, "same", []), rep) - # a third change: never more than two certificates — the new one and the one that was first - f3 = TFake(source=CA_PEM, copy=f.tree[osapply.PVE_CA_COPY]) - rc, rep = run(f3) + self.assertEqual(f.openssl_stdin, [CA_PEM_2.decode()], "the SOURCE is checked in full; the kept one by its own check") + + def test_the_previous_root_stays_only_for_the_changeover(self): + old = osapply.PEM_CERT_RE.findall(CA_PEM)[0] + bundle = CA_PEM_2 + old + b"\n" + # Proxmox still serves the OLD certificate: nothing is written, the old root stays + f = TFake(source=CA_PEM_2, copy=bundle) + rc, rep = run(f) + self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], f.puts), (0, "same", []), rep) + self.assertTrue(rep["pve_ca_copy"]["kept_previous"]) + self.assertEqual(len(f.handshakes), 1) + self.assertIn(osapply.PVE_CA_SOURCE, f.handshakes[0], "the handshake is verified with the NEW root alone (the source file)") + self.assertIn("-no-CApath", f.handshakes[0]) + self.assertIn("-verify_return_error", f.handshakes[0]) + # pveproxy was reloaded and serves a certificate of the NEW root: the old root is dropped + f = TFake(source=CA_PEM_2, copy=bundle) + f.serves_new = True + rc, rep = run(f) self.assertEqual(rc, 0, rep) - self.assertEqual(osapply.PEM_CERT_RE.findall(f3.tree[osapply.PVE_CA_COPY]), - osapply.PEM_CERT_RE.findall(CA_PEM) + osapply.PEM_CERT_RE.findall(CA_PEM_2)) - # a first copy keeps nothing behind it + self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2, "after the changeover the copy is the new root alone") + self.assertEqual((rep["pve_ca_copy"]["outcome"], rep["pve_ca_copy"]["dropped_previous"], rep["pve_ca_copy"]["kept_previous"]), ("written", True, False)) + # the old root has ENDED while Proxmox still serves the old certificate: it verifies nothing, so it is dropped + f = TFake(source=CA_PEM_2, copy=bundle) + f.dead_ca = {old.decode()} + rc, rep = run(f) + self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2) + # an ended previous root is not kept at the change itself either + f = TFake(source=CA_PEM_2, copy=CA_PEM) + f.dead_ca = {old.decode()} + rc, rep = run(f) + self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2) + self.assertFalse(rep["pve_ca_copy"]["kept_previous"]) + + def test_a_copy_with_anything_unchecked_in_it_is_rewritten(self): + old = osapply.PEM_CERT_RE.findall(CA_PEM)[0] + for name, copy in (("text after the source", CA_PEM_2 + b"anything at all\n"), + ("a third certificate", CA_PEM_2 + old + b"\n" + old + b"\n"), + ("a key in the file", CA_PEM_2 + b"-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n"), + ("the two in the wrong order", CA_PEM + osapply.PEM_CERT_RE.findall(CA_PEM_2)[0] + b"\n")): + f = TFake(source=CA_PEM_2, copy=copy) + f.serves_new = True + rc, rep = run(f) + self.assertEqual(rc, 0, (name, rep)) + got = f.tree[osapply.PVE_CA_COPY] + self.assertTrue(got.startswith(CA_PEM_2), name) + self.assertNotIn(b"anything", got, name) + self.assertNotIn(b"PRIVATE KEY", got, name) + self.assertLessEqual(len(osapply.PEM_CERT_RE.findall(got)), 2, name) + # a first copy keeps nothing behind it, and makes no handshake f4 = TFake() rc, rep = run(f4) self.assertFalse(rep["pve_ca_copy"]["kept_previous"]) - self.assertEqual(f4.tree[osapply.PVE_CA_COPY], CA_PEM) + self.assertEqual((f4.tree[osapply.PVE_CA_COPY], f4.handshakes), (CA_PEM, [])) def test_a_source_that_is_not_a_ca_certificate_is_refused(self): self.refused(TFake(ca=False), "T3") # a leaf certificate @@ -2766,6 +2811,42 @@ class PVECACopy(unittest.TestCase): self.assertEqual(got, want, f"{name}: {rep}") if want == "written": self.assertIn(rep["pve_ca_copy"]["days_left"], (29, 30)) + # the two checks the changeover uses, with the REAL openssl and a REAL TLS server + ended = True + try: # an ENDED CA needs openssl 3.4+ (-not_after); older: that one check is not made + sh("openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", "old.key", "-out", "old.pem", + "-not_before", "20190101000000Z", "-not_after", "20200101000000Z", + "-subj", "/CN=ended CA", "-addext", "basicConstraints=critical,CA:TRUE") + except subprocess.CalledProcessError: + ended = False + sh("openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", "srv.key", "-out", "srv.csr", "-subj", "/CN=pve") + sh("openssl", "x509", "-req", "-in", "srv.csr", "-CA", "ca.pem", "-CAkey", "ca.key", "-CAcreateserial", "-out", "srv.pem", "-days", "5") + app = osapply.Apply.__new__(osapply.Apply) + app.r = real + block = lambda n: osapply.PEM_CERT_RE.findall(open(os.path.join(d, n), "rb").read())[0] + self.assertTrue(app.is_live_ca(block("ca.pem")), "a CA that has not ended") + self.assertFalse(app.is_live_ca(block("leaf.pem")), "a leaf is not a CA") + if ended: + self.assertFalse(app.is_live_ca(block("old.pem")), "an ended CA is not a live one") + import socket + s0 = socket.socket(); s0.bind(("127.0.0.1", 0)); port = s0.getsockname()[1]; s0.close() + srv = subprocess.Popen(["openssl", "s_server", "-accept", f"127.0.0.1:{port}", "-cert", "srv.pem", "-key", "srv.key", "-quiet"], + cwd=d, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + saved, osapply.PVE_LOCAL_API = osapply.PVE_LOCAL_API, f"127.0.0.1:{port}" + ok = False + for _ in range(40): # the server needs a moment to listen + if app.pve_serves_with(os.path.join(d, "ca.pem")): + ok = True + break + __import__("time").sleep(0.1) + self.assertTrue(ok, "the server's certificate is signed by ca.pem: the handshake must verify with it alone") + self.assertFalse(app.pve_serves_with(os.path.join(d, "leaf.pem")), "another certificate as the CA file must NOT verify") + osapply.PVE_LOCAL_API = "127.0.0.1:1" + self.assertFalse(app.pve_serves_with(os.path.join(d, "ca.pem")), "no answer is a no") + finally: + osapply.PVE_LOCAL_API = saved + srv.kill(); srv.wait() finally: shutil.rmtree(d, ignore_errors=True)