agent (unreleased): the CA copy keeps its previous root only for the changeover, and is compared exactly (the commit's security review)

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

CHANGELOG.md

@@ -11,8 +11,11 @@ Commit only. Each part can be released alone; none needs release 2b. No `configs
   copy), and its root step has a 90-second deadline (it runs on the start path and inside the report loop). (3) **The
   CA copy keeps its previous certificate behind the new one** (`configs/felhom-os-apply`, mode `pve-ca-copy`): the
   copy was replaced before anything could be proven with the new root, and Proxmox serves its old certificate until
-  pveproxy is reloaded — an agent that restarted in that window had no root that verified it. At most two
+  pveproxy is reloaded — an agent that restarted in that window had no root that verified it. **Only for the
-  certificates; the same source is still „same”. `configs/` changes again, so the release needs both signed jobs.
+  changeover:** at every later run the step makes one handshake to the local Proxmox with the NEW root alone, and
+  drops the previous root as soon as that verifies, or when the previous root is no longer a CA that has not ended.
+  The copy is compared exactly, so nothing the step did not check stays in it. To drop the old root at once by hand
+  (a root changed because it leaked): delete the copy. `configs/` changes again, so the release needs both signed jobs.
   (4) The profile-refresh stanza is built from the wrapper's ANSWER (its record of the last refresh comes back at
   every ask); the record file is a bonus — its folder is root-only on a box where a signed job made it first (read in
   source, not on a box). `rewritten` is absent when not known. (5) A watchdog give-up is forgotten when the guest is

configs/felhom-os-apply

@@ -167,6 +167,7 @@ KERNEL_ACTIVE = ("staged", "oneshot", "judging", "reverting")
 PVE_CA_SOURCE = "/etc/pve/pve-root-ca.pem"
 PVE_CA_COPY = "/etc/felhom-agent/pve-root-ca.pem"      # 0644 root: a CA certificate is public; the agent only reads it
 PVE_CA_MAX_BYTES = 64 * 1024
+PVE_LOCAL_API = "127.0.0.1:8006"                        # pveproxy on this host (the installer's proof uses the same)
 PEM_CERT_RE = re.compile(rb"-----BEGIN CERTIFICATE-----\s[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----")
 # The security-profile refresh (R-1388, R-1301, `11` §5.12; measured on demo-hp 2026-11-14,
 # felhom.eu audits/profile-refresh-spike-2026-11-14/). AppArmor keeps one cache folder per kernel feature set; a
@@ -1213,11 +1214,30 @@ class Apply:
         return 0
 
     # ---------- trust in Proxmox by a copy of its root CA (R-1380, D80) ----------
+    def is_live_ca(self, block):
+        """One PEM certificate block: is it a CA certificate that has not ended? (openssl, the bytes on stdin)."""
+        try:
+            rc, out, _ = self.r.host(["openssl", "x509", "-noout", "-ext", "basicConstraints", "-checkend", "0"], 30,
+                                     stdin=block.decode("ascii") + "\n")
+        except (UnicodeDecodeError, subprocess.SubprocessError, OSError):
+            return False
+        return rc == 0 and re.search(r"\bCA:TRUE\b", out) is not None
+
+    def pve_serves_with(self, cafile):
+        """Does the local Proxmox API serve a certificate that verifies with this CA file ALONE? One handshake to
+        PVE_LOCAL_API; nothing is sent. Any doubt (no answer, a timeout) is "no" — the previous root then stays."""
+        try:
+            rc, out, err = self.r.host(["openssl", "s_client", "-connect", PVE_LOCAL_API, "-CAfile", cafile, "-no-CApath",
+                                        "-verify_return_error"], 20, stdin="")
+        except (subprocess.SubprocessError, OSError):
+            return False
+        return rc == 0 and "Verify return code: 0 (ok)" in (out + err)
+
     def pve_ca_copy(self):
         """Copy exactly PVE_CA_SOURCE to exactly PVE_CA_COPY — only if the source is ONE CA certificate that has not
         ended — written whole or not at all (a root-owned temp file beside the target, fsync, rename). "same" when the
-        copy already starts with the source's bytes. The bytes that are checked are the bytes that are written first;
+        copy is exactly the source's bytes. The bytes that are checked are the bytes that are written first; only
-        behind them the copy keeps its own previous certificate (see below), so the copy holds one or two."""
+        during a change of the root the copy also holds its previous certificate behind them (see below)."""
         r = self.r
         res = {"outcome": None, "path": PVE_CA_COPY}
         self.report["pve_ca_copy"] = res
@@ -1254,18 +1274,30 @@ class Apply:
             have = r.read_bytes(PVE_CA_COPY)
         except OSError:
             have = None
-        if have is not None and have.startswith(data):
+        if have == data:
-            res["outcome"] = "same"          # the source's certificate is already first in the copy
+            res["outcome"] = "same"
+            return 0
+        # The copy the agent trusts by is replaced BEFORE anything can be proven with the new root (Proxmox serves its
+        # old certificate until pveproxy is reloaded). So at a CHANGE the copy's previous first certificate — written
+        # by this same step — is kept behind the new one, and the agent trusts both roots across the change, also
+        # after a restart. That second root is kept ONLY for the changeover: it is dropped as soon as Proxmox serves a
+        # certificate that verifies with the new root alone, and at once when it is no longer a CA that has not ended.
+        # The copy is compared EXACTLY: nothing this step did not check is ever left in it. (Review, 2026-11-14.)
+        # To drop the old root at once by hand (a root that was changed because it leaked): delete the copy.
+        prev = PEM_CERT_RE.findall(have) if have and b"PRIVATE KEY" not in have else []
+        if len(prev) == 2 and have == data + prev[1] + b"\n":
+            over = self.pve_serves_with(PVE_CA_SOURCE)
+            if not over and self.is_live_ca(prev[1]):
+                res["outcome"], res["kept_previous"] = "same", True   # the changeover is still on
+                return 0
+            r.put_file(PVE_CA_COPY, data, 0o644)
+            res["outcome"], res["replaced"], res["kept_previous"], res["dropped_previous"] = "written", True, False, True
+            r.log(f"os-apply: PVE CA COPY the previous root is dropped from {PVE_CA_COPY} "
+                  f"({'Proxmox serves a certificate of the new root' if over else 'the previous root is no longer a live CA'})")
             return 0
-        # The copy the agent trusts by is replaced BEFORE anything can be proven with the new one (Proxmox serves its
-        # old certificate until pveproxy is reloaded). So the copy's previous FIRST certificate — written by this same
-        # step, so already checked as a CA — is kept behind the new one: the agent then trusts both roots across the
-        # change, also after a restart. At most two certificates; an ended one verifies nothing. (Review, 2026-11-14.)
         want, kept = data, False
-        if have:
+        if prev and prev[0].strip() != certs[0].strip() and self.is_live_ca(prev[0]):
-            prev = PEM_CERT_RE.findall(have)
+            want, kept = data + prev[0] + b"\n", True
-            if prev and b"PRIVATE KEY" not in have and prev[0].strip() != certs[0].strip():
-                want, kept = data + prev[0] + b"\n", True
         r.put_file(PVE_CA_COPY, want, 0o644)
         res["outcome"], res["replaced"], res["kept_previous"] = "written", have is not None, kept
         r.log(f"os-apply: PVE CA COPY written {PVE_CA_COPY} sha256={res['sha256'][:16]} ends={res['not_after']} "

configs/test_felhom_os_apply.py

@@ -2648,6 +2648,9 @@ class TFake(Fake):
         if copy is not None:
             self.tree[osapply.PVE_CA_COPY] = copy
         self.openssl_stdin, self.puts = [], []
+        self.serves_new = False        # does pveproxy serve a certificate that verifies with the NEW root alone?
+        self.dead_ca = set()           # certificate texts openssl says are not a live CA (ended)
+        self.handshakes = []
         self.plan = {"release_id": "pve-ca", "layer": "host", "mode": "pve-ca-copy"}
 
     def read_bytes(self, p):
@@ -2664,6 +2667,12 @@ class TFake(Fake):
         self.tree[path] = data
 
     def host(self, argv, timeout=600, stdin=None):
+        if argv[0] == "openssl" and argv[1] == "s_client":
+            self.handshakes.append(argv)
+            return (0, "Verify return code: 0 (ok)\n", "") if self.serves_new else (1, "", "Verify return code: 19 (self-signed certificate in certificate chain)\n")
+        if argv[0] == "openssl" and "-checkend" in argv:
+            return (1, "X509v3 Basic Constraints: critical\n    CA:TRUE\nCertificate will expire\n", "") if stdin.strip() in self.dead_ca else \
+                (0, "X509v3 Basic Constraints: critical\n    CA:TRUE\nCertificate will not expire\n", "")
         if argv[0] == "openssl":
             self.calls.append(("host", argv))
             self.openssl_stdin.append(stdin)
@@ -2704,26 +2713,62 @@ class PVECACopy(unittest.TestCase):
         self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], rep["pve_ca_copy"]["replaced"]), (0, "written", True), rep)
         # the new certificate FIRST, and the copy's previous one kept behind it: Proxmox serves its old certificate until
         # pveproxy is reloaded, and an agent that restarts in that window must still have a root that verifies it
-        self.assertTrue(f.tree[osapply.PVE_CA_COPY].startswith(CA_PEM_2))
+        bundle = f.tree[osapply.PVE_CA_COPY]
-        self.assertEqual(osapply.PEM_CERT_RE.findall(f.tree[osapply.PVE_CA_COPY]),
+        self.assertEqual(bundle, CA_PEM_2 + osapply.PEM_CERT_RE.findall(CA_PEM)[0] + b"\n")
-                         osapply.PEM_CERT_RE.findall(CA_PEM_2) + osapply.PEM_CERT_RE.findall(CA_PEM))
         self.assertTrue(rep["pve_ca_copy"]["kept_previous"])
-        self.assertEqual(f.openssl_stdin, [CA_PEM_2.decode()], "only the SOURCE is checked by openssl")
+        self.assertEqual(f.openssl_stdin, [CA_PEM_2.decode()], "the SOURCE is checked in full; the kept one by its own check")
-        # the same source again: nothing is written (the copy already starts with it)
+
-        f2 = TFake(source=CA_PEM_2, copy=f.tree[osapply.PVE_CA_COPY])
+    def test_the_previous_root_stays_only_for_the_changeover(self):
-        rc, rep = run(f2)
+        old = osapply.PEM_CERT_RE.findall(CA_PEM)[0]
-        self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], f2.puts), (0, "same", []), rep)
+        bundle = CA_PEM_2 + old + b"\n"
-        # a third change: never more than two certificates — the new one and the one that was first
+        # Proxmox still serves the OLD certificate: nothing is written, the old root stays
-        f3 = TFake(source=CA_PEM, copy=f.tree[osapply.PVE_CA_COPY])
+        f = TFake(source=CA_PEM_2, copy=bundle)
-        rc, rep = run(f3)
+        rc, rep = run(f)
+        self.assertEqual((rc, rep["pve_ca_copy"]["outcome"], f.puts), (0, "same", []), rep)
+        self.assertTrue(rep["pve_ca_copy"]["kept_previous"])
+        self.assertEqual(len(f.handshakes), 1)
+        self.assertIn(osapply.PVE_CA_SOURCE, f.handshakes[0], "the handshake is verified with the NEW root alone (the source file)")
+        self.assertIn("-no-CApath", f.handshakes[0])
+        self.assertIn("-verify_return_error", f.handshakes[0])
+        # pveproxy was reloaded and serves a certificate of the NEW root: the old root is dropped
+        f = TFake(source=CA_PEM_2, copy=bundle)
+        f.serves_new = True
+        rc, rep = run(f)
         self.assertEqual(rc, 0, rep)
-        self.assertEqual(osapply.PEM_CERT_RE.findall(f3.tree[osapply.PVE_CA_COPY]),
+        self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2, "after the changeover the copy is the new root alone")
-                         osapply.PEM_CERT_RE.findall(CA_PEM) + osapply.PEM_CERT_RE.findall(CA_PEM_2))
+        self.assertEqual((rep["pve_ca_copy"]["outcome"], rep["pve_ca_copy"]["dropped_previous"], rep["pve_ca_copy"]["kept_previous"]), ("written", True, False))
-        # a first copy keeps nothing behind it
+        # the old root has ENDED while Proxmox still serves the old certificate: it verifies nothing, so it is dropped
+        f = TFake(source=CA_PEM_2, copy=bundle)
+        f.dead_ca = {old.decode()}
+        rc, rep = run(f)
+        self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2)
+        # an ended previous root is not kept at the change itself either
+        f = TFake(source=CA_PEM_2, copy=CA_PEM)
+        f.dead_ca = {old.decode()}
+        rc, rep = run(f)
+        self.assertEqual(f.tree[osapply.PVE_CA_COPY], CA_PEM_2)
+        self.assertFalse(rep["pve_ca_copy"]["kept_previous"])
+
+    def test_a_copy_with_anything_unchecked_in_it_is_rewritten(self):
+        old = osapply.PEM_CERT_RE.findall(CA_PEM)[0]
+        for name, copy in (("text after the source", CA_PEM_2 + b"anything at all\n"),
+                           ("a third certificate", CA_PEM_2 + old + b"\n" + old + b"\n"),
+                           ("a key in the file", CA_PEM_2 + b"-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n"),
+                           ("the two in the wrong order", CA_PEM + osapply.PEM_CERT_RE.findall(CA_PEM_2)[0] + b"\n")):
+            f = TFake(source=CA_PEM_2, copy=copy)
+            f.serves_new = True
+            rc, rep = run(f)
+            self.assertEqual(rc, 0, (name, rep))
+            got = f.tree[osapply.PVE_CA_COPY]
+            self.assertTrue(got.startswith(CA_PEM_2), name)
+            self.assertNotIn(b"anything", got, name)
+            self.assertNotIn(b"PRIVATE KEY", got, name)
+            self.assertLessEqual(len(osapply.PEM_CERT_RE.findall(got)), 2, name)
+        # a first copy keeps nothing behind it, and makes no handshake
         f4 = TFake()
         rc, rep = run(f4)
         self.assertFalse(rep["pve_ca_copy"]["kept_previous"])
-        self.assertEqual(f4.tree[osapply.PVE_CA_COPY], CA_PEM)
+        self.assertEqual((f4.tree[osapply.PVE_CA_COPY], f4.handshakes), (CA_PEM, []))
 
     def test_a_source_that_is_not_a_ca_certificate_is_refused(self):
         self.refused(TFake(ca=False), "T3")                                  # a leaf certificate
@@ -2766,6 +2811,42 @@ class PVECACopy(unittest.TestCase):
                 self.assertEqual(got, want, f"{name}: {rep}")
                 if want == "written":
                     self.assertIn(rep["pve_ca_copy"]["days_left"], (29, 30))
+            # the two checks the changeover uses, with the REAL openssl and a REAL TLS server
+            ended = True
+            try:                                        # an ENDED CA needs openssl 3.4+ (-not_after); older: that one check is not made
+                sh("openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", "old.key", "-out", "old.pem",
+                   "-not_before", "20190101000000Z", "-not_after", "20200101000000Z",
+                   "-subj", "/CN=ended CA", "-addext", "basicConstraints=critical,CA:TRUE")
+            except subprocess.CalledProcessError:
+                ended = False
+            sh("openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", "srv.key", "-out", "srv.csr", "-subj", "/CN=pve")
+            sh("openssl", "x509", "-req", "-in", "srv.csr", "-CA", "ca.pem", "-CAkey", "ca.key", "-CAcreateserial", "-out", "srv.pem", "-days", "5")
+            app = osapply.Apply.__new__(osapply.Apply)
+            app.r = real
+            block = lambda n: osapply.PEM_CERT_RE.findall(open(os.path.join(d, n), "rb").read())[0]
+            self.assertTrue(app.is_live_ca(block("ca.pem")), "a CA that has not ended")
+            self.assertFalse(app.is_live_ca(block("leaf.pem")), "a leaf is not a CA")
+            if ended:
+                self.assertFalse(app.is_live_ca(block("old.pem")), "an ended CA is not a live one")
+            import socket
+            s0 = socket.socket(); s0.bind(("127.0.0.1", 0)); port = s0.getsockname()[1]; s0.close()
+            srv = subprocess.Popen(["openssl", "s_server", "-accept", f"127.0.0.1:{port}", "-cert", "srv.pem", "-key", "srv.key", "-quiet"],
+                                   cwd=d, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            try:
+                saved, osapply.PVE_LOCAL_API = osapply.PVE_LOCAL_API, f"127.0.0.1:{port}"
+                ok = False
+                for _ in range(40):                     # the server needs a moment to listen
+                    if app.pve_serves_with(os.path.join(d, "ca.pem")):
+                        ok = True
+                        break
+                    __import__("time").sleep(0.1)
+                self.assertTrue(ok, "the server's certificate is signed by ca.pem: the handshake must verify with it alone")
+                self.assertFalse(app.pve_serves_with(os.path.join(d, "leaf.pem")), "another certificate as the CA file must NOT verify")
+                osapply.PVE_LOCAL_API = "127.0.0.1:1"
+                self.assertFalse(app.pve_serves_with(os.path.join(d, "ca.pem")), "no answer is a no")
+            finally:
+                osapply.PVE_LOCAL_API = saved
+                srv.kill(); srv.wait()
         finally:
             shutil.rmtree(d, ignore_errors=True)