From 83e61671453b78553109fb91e9f33e009cfe89ee Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 8 Oct 2026 09:58:00 +0200 Subject: [PATCH] =?UTF-8?q?ep0-copy-gc:=20fail-safe=20guards=20(partial=20?= =?UTF-8?q?ep0=20list=20aborts,=20one=20deletion=20per=20run,=20HOLD=20fil?= =?UTF-8?q?e)=20=E2=80=94=20from=20the=20commit=20security=20review;=20sti?= =?UTF-8?q?ll=20not=20installed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- documentation/runbooks/ep0-datastore-copy.md | 8 +++++- scripts/ep0-copy-gc/felhom-ep0-copy-gc | 21 ++++++++++++++-- scripts/ep0-copy-gc/test_ep0_copy_gc.py | 26 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/documentation/runbooks/ep0-datastore-copy.md b/documentation/runbooks/ep0-datastore-copy.md index ebb34f44..3bba480e 100644 --- a/documentation/runbooks/ep0-datastore-copy.md +++ b/documentation/runbooks/ep0-datastore-copy.md @@ -72,6 +72,10 @@ rebuild an endpoint and pull back, so every host reconnects the usual way. ### Route 2 — rebuild the endpoint. Not walked. +**First, on DooPlex: `sudo touch /etc/felhom/ep0-copy-gc/HOLD`** (when the removal job is installed) — a rebuilt or +half-restored ep0 lists fewer namespaces, and the job must not read that as customer deletions. Remove the file when ep0 +lists every customer again. + Provision a new ep0 (06 §5), then on it add DooPlex as a remote and run `proxmox-backup-manager pull ep0-copy felhom-offsite`. Every box then reconnects as before. @@ -84,7 +88,9 @@ Ruling: removed **within 30 days**. The job `scripts/ep0-copy-gc/felhom-ep0-copy 08:00: a top-level namespace in the copy that ep0 no longer lists is remembered with the day it was first seen absent; after **7 days** absent it is deleted from the copy (`proxmox-backup-client namespace delete --delete-groups true`); one that reappears is forgotten; `operator` is never deleted. If ep0's list cannot be read, or reads empty, it does -nothing. Without `--apply` it only prints. Worst case: ≤ 1 day to notice + 7 days grace + 1 day = inside 30 days. +nothing; if more than 2 namespaces are absent at once (a partial list, e.g. a rebuilt ep0) it does nothing; it +deletes at most one namespace per run; a HOLD file (`/etc/felhom/ep0-copy-gc/HOLD`) stops it. Without `--apply` it +only prints. Worst case: ≤ 1 day to notice + 7 days grace + 1 day = inside 30 days. **Not installed. To install (needs the operator's word — it is a change on DooPlex and it deletes data):** 1. Two tokens, secrets in root-only files under `/etc/felhom/ep0-copy-gc/` (0600), never printed: diff --git a/scripts/ep0-copy-gc/felhom-ep0-copy-gc b/scripts/ep0-copy-gc/felhom-ep0-copy-gc index af3bf408..0143c16e 100755 --- a/scripts/ep0-copy-gc/felhom-ep0-copy-gc +++ b/scripts/ep0-copy-gc/felhom-ep0-copy-gc @@ -12,8 +12,9 @@ The rule, in one place (`decide`): * KEEP (`operator`, the hub database's copies) is never deleted. With the daily timer: deletion on ep0 → seen absent within a day → deleted 7 days later, inside the 30-day line. -Fail-safe: if ep0's list cannot be read, or reads EMPTY, nothing is recorded and nothing is deleted (an empty answer -is „could not tell", never „everything was deleted"). Default mode is a DRY RUN that only prints; `--apply` deletes. +Fail-safe: if ep0's list cannot be read, or reads EMPTY, or more than MAX_ABSENT (2) copy namespaces are absent at +once (a partial list — a rebuilt ep0 — not deletions), nothing is recorded and nothing is deleted; at most one deletion +per run; a HOLD file (/etc/felhom/ep0-copy-gc/HOLD) stops it during any ep0 recovery. Default mode is a DRY RUN that only prints; `--apply` deletes. Secrets: the two PBS token secrets are read from root-only files into the child's environment (PBS_PASSWORD); never printed. Runbook: documentation/runbooks/ep0-datastore-copy.md, „Removing a deleted customer's copy". @@ -35,6 +36,12 @@ LOCAL_REPO = os.environ.get("LOCAL_REPO", "root@pam!ep0-copy-gc@localhost:ep0-co LOCAL_TOKEN_FILE = os.environ.get("LOCAL_TOKEN_FILE", "/etc/felhom/ep0-copy-gc/local-gc.secret") GRACE_DAYS = int(os.environ.get("EP0_COPY_GC_GRACE_DAYS", "7")) KEEP = {"operator"} +# Fail-safe guards (security review 2026-10-08): a PARTIAL ep0 list (a rebuilt or half-restored ep0, a token that sees +# less) would read as „these customers were deleted". So: more than MAX_ABSENT absent at once → ABORT, nothing recorded; +# at most MAX_DELETE deletions per run; and a HOLD file stops everything (create it during any ep0 recovery). +MAX_ABSENT = int(os.environ.get("EP0_COPY_GC_MAX_ABSENT", "2")) +MAX_DELETE = 1 +HOLD_FILE = os.environ.get("EP0_COPY_GC_HOLD", "/etc/felhom/ep0-copy-gc/HOLD") def log(msg): @@ -83,6 +90,9 @@ def main(argv=None): ap.add_argument("--apply", action="store_true", help="delete; without it, only print what would be deleted") a = ap.parse_args(argv) today = dt.date.today() + if os.path.exists(HOLD_FILE): + log("HOLD — %s exists (an ep0 recovery in progress?); nothing recorded, nothing deleted" % HOLD_FILE) + return 0 copy_ns = {d for d in os.listdir(COPY_NS_DIR) if os.path.isdir(os.path.join(COPY_NS_DIR, d))} try: ep0 = ep0_namespaces() @@ -98,6 +108,13 @@ def main(argv=None): except FileNotFoundError: state = {} to_delete, new_state = decide(copy_ns, ep0, state, today) + if len(new_state) > MAX_ABSENT: + log("ABORT — %d namespaces absent on ep0 at once (limit %d): ep0's list looks partial (a rebuild?), not like " + "customer deletions; nothing recorded, nothing deleted. Absent: %s" % (len(new_state), MAX_ABSENT, ", ".join(sorted(new_state)))) + return 2 + if len(to_delete) > MAX_DELETE: + log("limit: %d due, deleting %d this run (the rest stay due)" % (len(to_delete), MAX_DELETE)) + to_delete = to_delete[:MAX_DELETE] for ns, first in sorted(new_state.items()): log("absent on ep0 since %s: %s" % (first, ns)) failed = 0 diff --git a/scripts/ep0-copy-gc/test_ep0_copy_gc.py b/scripts/ep0-copy-gc/test_ep0_copy_gc.py index 954b05c3..8c88434d 100644 --- a/scripts/ep0-copy-gc/test_ep0_copy_gc.py +++ b/scripts/ep0-copy-gc/test_ep0_copy_gc.py @@ -119,6 +119,32 @@ class EndToEnd(unittest.TestCase): self.assertEqual(self.deletes(), []) self.assertEqual(json.load(open(self.state)), {"gone-cust": "2026-01-01"}) # state untouched + def test_mass_absence_aborts(self): + # ep0 rebuilt / half-restored: it lists only demo-hp; three copy namespaces look „deleted" → abort, delete none. + for n in ("c2", "c3"): + os.makedirs(os.path.join(self.ns, n)) + self.seed("gone-cust", "2026-01-01") + r = self.run_gc("--apply") + self.assertEqual(r.returncode, 2, r.stdout) + self.assertEqual(self.deletes(), []) + self.assertEqual(json.load(open(self.state)), {"gone-cust": "2026-01-01"}) # state untouched + + def test_one_deletion_per_run(self): + os.makedirs(os.path.join(self.ns, "c2")) + os.makedirs(os.path.dirname(self.state), exist_ok=True) + json.dump({"gone-cust": "2026-01-01", "c2": "2026-01-01"}, open(self.state, "w")) + r = self.run_gc("--apply") + self.assertEqual(r.returncode, 0, r.stdout) + self.assertEqual(len(self.deletes()), 1) + + def test_hold_file_stops_everything(self): + hold = os.path.join(self.t, "HOLD"); open(hold, "w").write("") + self.seed("gone-cust", "2026-01-01") + r = self.run_gc("--apply", EP0_COPY_GC_HOLD=hold) + self.assertEqual(r.returncode, 0) + self.assertEqual(self.deletes(), []) + self.assertFalse(os.path.exists(self.log)) # not even ep0 was asked + def test_secret_never_printed(self): open(os.path.join(self.t, "ep0.secret"), "w").write("SEKRIT-VALUE") self.seed("gone-cust", "2026-01-01")