From 05becb04d00b07eb0d2744a06e0ff3fcf62c194d Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 8 Oct 2026 09:54:39 +0200 Subject: [PATCH] R-901: a deleted customer's audit rows go 1 year after the deletion (hub, unreleased); ep0-copy removal job written, not installed (decision 181); both times in the privacy draft Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../legal/DRAFT-adatkezelesi-tajekoztato.md | 13 +- documentation/runbooks/ep0-datastore-copy.md | 21 +++ hub/CHANGELOG.md | 9 +- hub/cmd/hub/main.go | 6 + hub/internal/store/deleted_customer_audit.go | 70 ++++++++++ .../store/deleted_customer_audit_test.go | 76 ++++++++++ scripts/ep0-copy-gc/felhom-ep0-copy-gc | 127 +++++++++++++++++ .../ep0-copy-gc/felhom-ep0-copy-gc.service | 9 ++ scripts/ep0-copy-gc/felhom-ep0-copy-gc.timer | 9 ++ scripts/ep0-copy-gc/test_ep0_copy_gc.py | 130 ++++++++++++++++++ 10 files changed, 463 insertions(+), 7 deletions(-) create mode 100644 hub/internal/store/deleted_customer_audit.go create mode 100644 hub/internal/store/deleted_customer_audit_test.go create mode 100755 scripts/ep0-copy-gc/felhom-ep0-copy-gc create mode 100644 scripts/ep0-copy-gc/felhom-ep0-copy-gc.service create mode 100644 scripts/ep0-copy-gc/felhom-ep0-copy-gc.timer create mode 100644 scripts/ep0-copy-gc/test_ep0_copy_gc.py diff --git a/documentation/legal/DRAFT-adatkezelesi-tajekoztato.md b/documentation/legal/DRAFT-adatkezelesi-tajekoztato.md index 19b0ef01..3e351168 100644 --- a/documentation/legal/DRAFT-adatkezelesi-tajekoztato.md +++ b/documentation/legal/DRAFT-adatkezelesi-tajekoztato.md @@ -117,7 +117,7 @@ A központi rendszer (`hub.felhom.eu`) a Felhom saját szerverén fut (k3s fürt | Alkalmazásnaplókból kiszűrt hibaüzenetek, előtte-utána 5 sor, kitakarással | hibaelhárítás | az utolsó előfordulás után **30 nap** | | Alkalmazásnapló-részlet, csak külön kérésre, kitakarással | hibaelhárítás | alkalmazásonként a **legutóbbi 2**, időkorlát nélkül; az ügyfél törlésekor törlődik | | Diagnosztikai naplócsomag, csak külön kérésre | hibaelhárítás | **72 óra** | -| Kiküldött értesítések naplója (esemény, szöveg, kézbesítés állapota) | elszámolhatóság | **nincs törlési idő; az ügyfél törlése után is megmarad** — lásd 11. pont | +| Kiküldött értesítések naplója (esemény, szöveg, kézbesítés állapota) | elszámolhatóság | az ügyfél törlése után **1 évig**, majd törlődik (az eseményekkel együtt); amíg az ügyfél aktív: [[ELLENŐRIZNI — megőrzési idő nincs meghatározva]] | | Az ügyfél-visszaállítás és a szervertörlés naplója | elszámolhatóság | **nincs törlési idő** | | **A mentés titkosító kulcsa, a háztartás helyreállító kódjával lezárva** („kulcsletét") | a mentés visszaállíthatósága gépcsere után | a szerver / ügyfél törléséig; a lecserélt régi kulcsok is megmaradnak, hogy a régi mentések nyithatók maradjanak | | A szerver vészhelyzeti konzoljelszava, titkosítva tárolva | üzemeltetés, hibaelhárítás | a szerver törléséig | @@ -149,7 +149,7 @@ nem tudja elolvasni (a kulcsot lásd az 5. pont „kulcsletét" sorában). |---|---|---|---| | Alkalmazásonkénti fájlmentés | **Hetzner Storage Box**, helyszínkód `fsn1`, ország: [[ELLENŐRIZNI]] | alkalmazásadatok, adatbázisok, megosztások | 7 napi, 4 heti, 6 havi példány; ügyfél-visszaállításkor (RESET) törlődik. A tárhely saját napi pillanatképeinek ideje: [[ELLENŐRIZNI]] | | Teljes szervermentés | **„ep0"** távoli mentőszerver, Hetzner Cloud, **Nürnberg (Németország)** | a teljes ügyfélkonténer | a legutóbbi 2 heti példány; az ügyfél törlésekor törlődik | -| Az ep0 másolata | a Felhom saját szerverén, ország: [[ELLENŐRIZNI]] | a fenti, továbbra is titkosítva | 8 heti példány — **a törlés után is**, lásd 11. pont | +| Az ep0 másolata | a Felhom saját szerverén, ország: [[ELLENŐRIZNI]] | a fenti, továbbra is titkosítva | 8 heti példány; az ügyfél törlése után **legfeljebb 30 napon belül** törlődik | @@ -231,10 +231,11 @@ bírósági jogérvényesítés: [[ÜGYVÉD TÖLTI KI]]. Resend, a Cloudflare és a Google adatfeldolgozóként kezeli. Javasolt új szöveg: `DRAFT-kapcsolat-hozzajarulas.md`. -5. **Törlés után megmaradó adatok.** A kiküldött értesítések naplója nem törlődik soha; a teljes - szervermentés Felhom-oldali másolata a törlés után is megtartja az utolsó 8 heti példányt, és - egyetlen dokumentum sem mondja, mikor törlődnek. Ezt a tájékoztató csak akkor ígérheti - másképp, ha a rendszer változik. +5. **Törlés után megmaradó adatok — DÖNTÉS 2026-10-08 (09:04):** az értesítési napló és az események a törlés után + 1 évig maradnak, majd törlődnek (a hub következő kiadásától); a teljes szervermentés Felhom-oldali másolata a + törlés után legfeljebb 30 napon belül törlődik — **ez a feladat 2026-10-08-án csak meg van írva, nincs + bekapcsolva**; a közzététel előtt ellenőrizni kell, hogy fut. Az aktív ügyfél értesítési naplójára továbbra sincs + megőrzési idő. ## 12. Az ügyvédnek ellenőrizni (R-802) — és ahol a vázlat találgatott diff --git a/documentation/runbooks/ep0-datastore-copy.md b/documentation/runbooks/ep0-datastore-copy.md index 33fce2e4..ebb34f44 100644 --- a/documentation/runbooks/ep0-datastore-copy.md +++ b/documentation/runbooks/ep0-datastore-copy.md @@ -77,6 +77,27 @@ Provision a new ep0 (06 §5), then on it add DooPlex as a remote and run Do **not** prune the copy tighter than decision 71 (8 weekly copies); never tighter than ep0's own retention. +## Removing a deleted customer's copy (R-901, `09` §3 decision 181) — WRITTEN 2026-10-08, NOT INSTALLED + +The sync keeps what ep0 removed (`remove-vanished false`), so a deleted customer's namespace stayed here for ever. +Ruling: removed **within 30 days**. The job `scripts/ep0-copy-gc/felhom-ep0-copy-gc` (tests beside it) runs daily at +08:00: a top-level namespace in the copy that ep0 no longer lists is remembered with the day it was first seen absent; +after **7 days** absent it is deleted from the copy (`proxmox-backup-client namespace delete --delete-groups true`); +one that reappears is forgotten; `operator` is never deleted. If ep0's list cannot be read, or reads empty, it does +nothing. Without `--apply` it only prints. Worst case: ≤ 1 day to notice + 7 days grace + 1 day = inside 30 days. + +**Not installed. To install (needs the operator's word — it is a change on DooPlex and it deletes data):** +1. Two tokens, secrets in root-only files under `/etc/felhom/ep0-copy-gc/` (0600), never printed: + - `ep0-reader.secret` — the existing ep0 read token `root@pam!dooplex-sync` (its secret is in `remote.cfg`, base64); + `ep0.fingerprint` — ep0's pinned certificate fingerprint (also in `remote.cfg`). + - `local-gc.secret` — a NEW local token: `proxmox-backup-manager user generate-token root@pam ep0-copy-gc`, then + `proxmox-backup-manager acl update /datastore/ep0-copy DatastoreAdmin --auth-id 'root@pam!ep0-copy-gc'`. +2. `install -m 755 scripts/ep0-copy-gc/felhom-ep0-copy-gc /usr/local/sbin/`; the `.service` and `.timer` into + `/etc/systemd/system/`; `systemctl daemon-reload`. +3. **First run by hand, DRY**: `sudo /usr/local/sbin/felhom-ep0-copy-gc` — read which namespaces it would remove; then + `systemctl enable --now felhom-ep0-copy-gc.timer`. +4. The chunks are freed by the Sunday 08:30 garbage collection. + ## Remove `sudo proxmox-backup-manager sync-job remove ep0-felhom-offsite; … verify-job remove verify-ep0-copy; … prune-job remove prune-ep0-copy;` diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index be98f1f1..90476542 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,9 +1,16 @@ -## Unreleased (2026-10-08) — an alarm when a box never backs up off-site because its escrow is pending (R-243; `09` §3 decision 179) — ships with tomorrow's hub release +## Unreleased (2026-10-08) — an alarm when a box never backs up off-site because its escrow is pending (R-243; `09` §3 decision 179); a deleted customer's audit rows go after 1 year (R-901; decision 181) — ships with tomorrow's hub release **Operator action on deploy: none.** Expect ONE `offsite_escrow_pending` mail for **Tester 2** on the first sweep after the deploy: its latest report (2026-10-04) says off-site ON, escrow `pending`, no successful run ever — the state the operator believes it is in (decision 170). +- **R-901 (operator ruling 2026-10-08 09:04, decision 181):** after a customer is DELETED, its `events` and + `notification_log` rows are deleted 1 year after the deletion — a new daily step in `pruneAll` + (`store.PruneDeletedCustomerAudit`). A deletion is a `customer_resets` journal row with leg `customer_delete` = ok and + a completion stamp (a RESET is not one); only rows created at or before the completion go, so a re-used customer id + keeps what it made since; the journal row stays. Note: `events` are pruned for every customer at `max_days` (90) as + before, so the year bites `notification_log`. Test `TestR901_DeletedCustomerAuditGoesAfterOneYear` (red-proved: with + the step returning early, `deleted events=0 notification_log=0, want 2 and 2`). - **R-243:** `offsite_stale` deliberately ignores a box whose escrow is not `escrowed` (pending is the designed onboarding state), so a household that never does the escrow step — or a box held in `awaiting_recovery_key` — never backed up off-site and nothing fired. New operator-only `offsite_escrow_pending` (warning): off-site ON, escrow not done, no diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 64f07188..ce36b9ad 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -1029,6 +1029,12 @@ func pruneAll(s *store.Store, maxDays int, logger *log.Logger) { } else if n > 0 { logger.Printf("[INFO] Pruned %d stale app issues", n) } + // R-901 (`09` §3 decision 181): a deleted customer's audit rows go 1 year after the deletion. + if ev, nl, err := s.PruneDeletedCustomerAudit(time.Now().UTC(), store.DeletedCustomerAuditKeep); err != nil { + logger.Printf("[ERROR] Prune deleted customers' audit rows: %v", err) + } else if ev+nl > 0 { + logger.Printf("[INFO] Pruned a deleted customer's audit rows (older than 1 year after the deletion): events=%d notification_log=%d", ev, nl) + } } // runUnsealBoxSecrets is `felhom-hub -unseal-box-secrets` (R-879 roll-back). It installs the sealing key diff --git a/hub/internal/store/deleted_customer_audit.go b/hub/internal/store/deleted_customer_audit.go new file mode 100644 index 00000000..7afb64b7 --- /dev/null +++ b/hub/internal/store/deleted_customer_audit.go @@ -0,0 +1,70 @@ +package store + +import ( + "encoding/json" + "fmt" + "time" +) + +// R-901 (operator ruling 2026-10-08 09:04, `09` §3 decision 181): after a customer is DELETED, the hub's audit rows for +// it — `events` and `notification_log` — are kept 1 year, then deleted. +// +// The delete cascade (web/customer_delete.go) deliberately leaves both tables („the audit trail outlives every +// lifecycle tier"), and nothing ever pruned `notification_log`; `events` are pruned for EVERY customer at +// retention.max_days (90 on the live hub), so in practice the year bites `notification_log`. Both are covered here so the +// rule holds if max_days is ever raised. +// +// WHICH customers: the ones whose delete cascade COMPLETED — a `customer_resets` journal row with leg +// `customer_delete` = `ok` and a `completed_at` stamp (a RESET has no such leg and is not a deletion). WHICH rows: only +// those created AT OR BEFORE that completion — a customer id re-used after the deletion keeps everything it made since. +// The journal row itself stays (F-14 provenance: an id and timestamps, no personal data). +// +// Pinned by TestR901_* (deleted_customer_audit_test.go). +const DeletedCustomerAuditKeep = 365 * 24 * time.Hour + +// PruneDeletedCustomerAudit deletes the events and notification_log rows of customers whose deletion completed before +// now-keep. Returns the rows deleted per table. +func (s *Store) PruneDeletedCustomerAudit(now time.Time, keep time.Duration) (events, notifications int64, err error) { + rows, err := s.db.Query(`SELECT customer_id, completed_at, legs_json FROM customer_resets WHERE completed_at IS NOT NULL`) + if err != nil { + return 0, 0, err + } + type del struct { + id string + at time.Time + } + var dels []del + for rows.Next() { + var id, at, legs string + if err := rows.Scan(&id, &at, &legs); err != nil { + rows.Close() + return 0, 0, err + } + m := map[string]string{} + if json.Unmarshal([]byte(legs), &m) != nil || m["customer_delete"] != "ok" { + continue + } + t := parseSQLiteTime(at) + if t.IsZero() || now.Sub(t) <= keep { + continue + } + dels = append(dels, del{id, t.UTC()}) + } + rows.Close() + for _, d := range dels { + cutoff := d.at.Format("2006-01-02 15:04:05") + r, err := s.db.Exec(`DELETE FROM events WHERE customer_id = ? AND created_at <= ?`, d.id, cutoff) + if err != nil { + return events, notifications, fmt.Errorf("PruneDeletedCustomerAudit %s: events: %w", d.id, err) + } + n, _ := r.RowsAffected() + events += n + r, err = s.db.Exec(`DELETE FROM notification_log WHERE customer_id = ? AND created_at <= ?`, d.id, cutoff) + if err != nil { + return events, notifications, fmt.Errorf("PruneDeletedCustomerAudit %s: notification_log: %w", d.id, err) + } + n, _ = r.RowsAffected() + notifications += n + } + return events, notifications, nil +} diff --git a/hub/internal/store/deleted_customer_audit_test.go b/hub/internal/store/deleted_customer_audit_test.go new file mode 100644 index 00000000..e1d06c4f --- /dev/null +++ b/hub/internal/store/deleted_customer_audit_test.go @@ -0,0 +1,76 @@ +package store + +import ( + "testing" + "time" +) + +// R-901: a deleted customer's events and notification_log rows go 1 year after the deletion — and nothing else goes. +// RED-PROOF: return early from PruneDeletedCustomerAudit (the pre-R-901 state: nothing prunes notification_log) → +// the c-old rows survive → this FAILS on the first assertion. +func TestR901_DeletedCustomerAuditGoesAfterOneYear(t *testing.T) { + s := newTestStore(t) + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + ts := func(d time.Duration) string { return now.Add(-d).Format("2006-01-02 15:04:05") } + day := 24 * time.Hour + + journal := func(cid, legs string, completedAgo time.Duration) { + if _, err := s.db.Exec(`INSERT INTO customer_resets (customer_id, started_at, completed_at, legs_json) VALUES (?, ?, ?, ?)`, + cid, ts(completedAgo+time.Minute), ts(completedAgo), legs); err != nil { + t.Fatal(err) + } + } + row := func(cid string, ago time.Duration) { + if _, err := s.db.Exec(`INSERT INTO events (customer_id, event_type, severity, message, created_at) VALUES (?, 'x', 'info', 'm', ?)`, cid, ts(ago)); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO notification_log (customer_id, event_type, severity, message, status, created_at) VALUES (?, 'x', 'info', 'm', 'sent', ?)`, cid, ts(ago)); err != nil { + t.Fatal(err) + } + } + deleted := `{"hosts":"ok","residue":"ok","customer_delete":"ok"}` + journal("c-old", deleted, 400*day) // deleted 400 days ago → its audit rows go + journal("c-recent", deleted, 100*day) // deleted 100 days ago → kept + journal("c-reset", `{"hetzner":"ok","pbs":"ok","db_purge":"ok"}`, 400*day) // a RESET, not a deletion → kept + row("c-old", 500*day) + row("c-old", 401*day) + row("c-old", 10*day) // the id re-used AFTER the deletion → kept + row("c-recent", 200*day) + row("c-reset", 500*day) + row("c-live", 900*day) // never deleted → kept (events' own 90-day prune is not this function's business) + + ev, nl, err := s.PruneDeletedCustomerAudit(now, DeletedCustomerAuditKeep) + if err != nil { + t.Fatal(err) + } + if ev != 2 || nl != 2 { + t.Fatalf("deleted events=%d notification_log=%d, want 2 and 2 (c-old's two rows before its deletion)", ev, nl) + } + count := func(table, cid string) int { + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE customer_id = ?`, cid).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + for _, c := range []struct { + cid string + want int + }{{"c-old", 1}, {"c-recent", 1}, {"c-reset", 1}, {"c-live", 1}} { + for _, tb := range []string{"events", "notification_log"} { + if got := count(tb, c.cid); got != c.want { + t.Errorf("%s rows for %s = %d, want %d", tb, c.cid, got, c.want) + } + } + } + // The journal row (provenance) stays. + var j int + _ = s.db.QueryRow(`SELECT COUNT(*) FROM customer_resets WHERE customer_id = 'c-old'`).Scan(&j) + if j != 1 { + t.Fatalf("the deletion journal row must stay, got %d", j) + } + // Idempotent. + if ev, nl, _ := s.PruneDeletedCustomerAudit(now, DeletedCustomerAuditKeep); ev+nl != 0 { + t.Fatalf("a second run deleted %d+%d rows, want 0", ev, nl) + } +} diff --git a/scripts/ep0-copy-gc/felhom-ep0-copy-gc b/scripts/ep0-copy-gc/felhom-ep0-copy-gc new file mode 100755 index 00000000..af3bf408 --- /dev/null +++ b/scripts/ep0-copy-gc/felhom-ep0-copy-gc @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""felhom-ep0-copy-gc — remove a DELETED customer's namespace from DooPlex's ep0-copy (R-901, `09` §3 decision 181). + +DooPlex pulls ep0's `felhom-offsite` into `ep0-copy` with `remove-vanished false`, so a namespace the customer delete +cascade destroyed on ep0 stays on DooPlex for ever. The ruling: it is removed within 30 days. + +The rule, in one place (`decide`): + * a top-level namespace present in the copy and ABSENT from ep0's own list is recorded with the day it was first seen + absent (state file); + * one that is absent for GRACE_DAYS (7) is deleted from the copy, groups and all; + * one that reappears on ep0 is forgotten (a re-created customer, or a listing hiccup); + * KEEP (`operator`, the hub database's copies) is never deleted. +With the daily timer: deletion on ep0 → seen absent within a day → deleted 7 days later, inside the 30-day line. + +Fail-safe: if ep0's list cannot be read, or reads EMPTY, nothing is recorded and nothing is deleted (an empty answer +is „could not tell", never „everything was deleted"). Default mode is a DRY RUN that only prints; `--apply` deletes. + +Secrets: the two PBS token secrets are read from root-only files into the child's environment (PBS_PASSWORD); never +printed. Runbook: documentation/runbooks/ep0-datastore-copy.md, „Removing a deleted customer's copy". +Tests: test_ep0_copy_gc.py (fake proxmox-backup-client on PATH). +""" +import argparse +import datetime as dt +import json +import os +import subprocess +import sys + +COPY_NS_DIR = os.environ.get("EP0_COPY_NS_DIR", "/mnt/5_hdd/backup/ep0-copy/ns") +STATE = os.environ.get("EP0_COPY_GC_STATE", "/var/lib/felhom-ep0-copy-gc/absent.json") +EP0_REPO = os.environ.get("EP0_REPO", "root@pam!dooplex-sync@127.0.0.1:18007:felhom-offsite") +EP0_TOKEN_FILE = os.environ.get("EP0_TOKEN_FILE", "/etc/felhom/ep0-copy-gc/ep0-reader.secret") +EP0_FINGERPRINT_FILE = os.environ.get("EP0_FINGERPRINT_FILE", "/etc/felhom/ep0-copy-gc/ep0.fingerprint") +LOCAL_REPO = os.environ.get("LOCAL_REPO", "root@pam!ep0-copy-gc@localhost:ep0-copy") +LOCAL_TOKEN_FILE = os.environ.get("LOCAL_TOKEN_FILE", "/etc/felhom/ep0-copy-gc/local-gc.secret") +GRACE_DAYS = int(os.environ.get("EP0_COPY_GC_GRACE_DAYS", "7")) +KEEP = {"operator"} + + +def log(msg): + print("ep0-copy-gc: " + msg, flush=True) + + +def decide(copy_ns, ep0_ns, state, today, grace_days=GRACE_DAYS, keep=KEEP): + """Pure rule. Returns (to_delete, new_state). state: {ns: 'YYYY-MM-DD' first seen absent}.""" + new_state = {} + to_delete = [] + for ns in sorted(copy_ns): + if ns in keep or ns in ep0_ns: + continue + first = state.get(ns, today.isoformat()) + new_state[ns] = first + if (today - dt.date.fromisoformat(first)).days >= grace_days: + to_delete.append(ns) + return to_delete, new_state + + +def pbc(args, token_file, fingerprint_file=None): + env = dict(os.environ) + with open(token_file) as f: + env["PBS_PASSWORD"] = f.read().strip() + if fingerprint_file: + with open(fingerprint_file) as f: + env["PBS_FINGERPRINT"] = f.read().strip() + return subprocess.run(["proxmox-backup-client"] + args, env=env, capture_output=True, text=True, timeout=300) + + +def ep0_namespaces(): + r = pbc(["namespace", "list", "--repository", EP0_REPO, "--output-format", "json"], EP0_TOKEN_FILE, EP0_FINGERPRINT_FILE) + if r.returncode != 0: + raise RuntimeError("ep0 namespace list failed (rc %d): %s" % (r.returncode, r.stderr.strip()[-200:])) + out = set() + for item in json.loads(r.stdout or "[]"): + name = item.get("ns", "") if isinstance(item, dict) else str(item) + top = name.split("/")[0] + if top: + out.add(top) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("--apply", action="store_true", help="delete; without it, only print what would be deleted") + a = ap.parse_args(argv) + today = dt.date.today() + copy_ns = {d for d in os.listdir(COPY_NS_DIR) if os.path.isdir(os.path.join(COPY_NS_DIR, d))} + try: + ep0 = ep0_namespaces() + except Exception as e: # noqa: BLE001 — any failure means „could not tell" + log("ABORT — %s; nothing recorded, nothing deleted" % e) + return 2 + if not ep0: + log("ABORT — ep0 lists NO namespace (read as „could not tell\", never as „all deleted\"); nothing changed") + return 2 + try: + with open(STATE) as f: + state = json.load(f) + except FileNotFoundError: + state = {} + to_delete, new_state = decide(copy_ns, ep0, state, today) + for ns, first in sorted(new_state.items()): + log("absent on ep0 since %s: %s" % (first, ns)) + failed = 0 + for ns in to_delete: + if not a.apply: + log("DRY RUN — would delete namespace %s from ep0-copy (absent on ep0 since %s)" % (ns, new_state[ns])) + continue + r = pbc(["namespace", "delete", ns, "--delete-groups", "true", "--repository", LOCAL_REPO], LOCAL_TOKEN_FILE) + if r.returncode != 0: + failed += 1 + log("FAILED to delete namespace %s (rc %d): %s" % (ns, r.returncode, r.stderr.strip()[-200:])) + continue + log("DELETED namespace %s from ep0-copy (absent on ep0 since %s)" % (ns, new_state[ns])) + new_state.pop(ns, None) + os.makedirs(os.path.dirname(STATE), exist_ok=True) + tmp = STATE + ".tmp" + with open(tmp, "w") as f: + json.dump(new_state, f, indent=1, sort_keys=True) + os.replace(tmp, STATE) + log("done: %d copy namespace(s), %d on ep0, %d absent, %d to delete%s, %d failed" + % (len(copy_ns), len(ep0), len(new_state) + (len(to_delete) if a.apply else 0), len(to_delete), + "" if a.apply else " (dry run)", failed)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ep0-copy-gc/felhom-ep0-copy-gc.service b/scripts/ep0-copy-gc/felhom-ep0-copy-gc.service new file mode 100644 index 00000000..fc2624cb --- /dev/null +++ b/scripts/ep0-copy-gc/felhom-ep0-copy-gc.service @@ -0,0 +1,9 @@ +[Unit] +Description=Felhom: remove a deleted customer's namespace from ep0-copy (R-901, decision 181) +Documentation=https://gitea.dooplex.hu/admin/felhom.eu/src/branch/main/documentation/runbooks/ep0-datastore-copy.md +After=felhom-ep0-pbs-tunnel.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/sbin/felhom-ep0-copy-gc --apply diff --git a/scripts/ep0-copy-gc/felhom-ep0-copy-gc.timer b/scripts/ep0-copy-gc/felhom-ep0-copy-gc.timer new file mode 100644 index 00000000..a4e4ad0c --- /dev/null +++ b/scripts/ep0-copy-gc/felhom-ep0-copy-gc.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Felhom: ep0-copy deleted-customer removal — daily, after the 05:00 pull and the 07:30 prune + +[Timer] +OnCalendar=*-*-* 08:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/ep0-copy-gc/test_ep0_copy_gc.py b/scripts/ep0-copy-gc/test_ep0_copy_gc.py new file mode 100644 index 00000000..954b05c3 --- /dev/null +++ b/scripts/ep0-copy-gc/test_ep0_copy_gc.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Tests for felhom-ep0-copy-gc (R-901). No PBS is reached: `proxmox-backup-client` is a fake on PATH that answers +`namespace list` from FAKE_EP0_NS and records every `namespace delete`. Each test asserts the CONSEQUENCE: which +namespace was deleted from the copy, and that nothing is deleted when ep0's answer cannot be trusted. +Run: python3 scripts/ep0-copy-gc/test_ep0_copy_gc.py""" +import datetime as dt +import importlib.machinery +import importlib.util +import json +import os +import stat +import subprocess +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "felhom-ep0-copy-gc") + +FAKE = r'''#!/usr/bin/env python3 +import json, os, sys +a = sys.argv[1:] +with open(os.environ["FAKE_LOG"], "a") as f: + f.write(" ".join(a) + " pw=" + ("set" if os.environ.get("PBS_PASSWORD") else "missing") + "\n") +if a[:2] == ["namespace", "list"]: + if os.environ.get("FAKE_EP0_FAIL"): + sys.stderr.write("connection refused\n"); sys.exit(255) + print(json.dumps([{"ns": n} for n in json.loads(os.environ["FAKE_EP0_NS"])])) + sys.exit(0) +if a[:2] == ["namespace", "delete"]: + sys.exit(0) +sys.exit(9) +''' + + +def load(): + loader = importlib.machinery.SourceFileLoader("gc", SCRIPT) + spec = importlib.util.spec_from_loader("gc", loader) + m = importlib.util.module_from_spec(spec) + loader.exec_module(m) + return m + + +class Decide(unittest.TestCase): + def test_rule(self): + m = load() + d = dt.date(2026, 10, 8) + state = {"old-cust": "2026-09-30", "fresh": "2026-10-07", "back": "2026-09-01"} + dele, new = m.decide({"old-cust", "fresh", "new-gone", "live", "operator", "back"}, + {"live", "back"}, state, d) + self.assertEqual(dele, ["old-cust"]) # absent 8 days → delete + self.assertEqual(new.get("fresh"), "2026-10-07") # absent 1 day → kept, remembered + self.assertEqual(new.get("new-gone"), "2026-10-08") # first seen absent today + self.assertNotIn("back", new) # reappeared on ep0 → forgotten + self.assertNotIn("operator", new) # KEEP is never a candidate + self.assertNotIn("live", new) + + +class EndToEnd(unittest.TestCase): + def setUp(self): + self.t = tempfile.mkdtemp() + bindir = os.path.join(self.t, "bin"); os.makedirs(bindir) + p = os.path.join(bindir, "proxmox-backup-client") + open(p, "w").write(FAKE); os.chmod(p, os.stat(p).st_mode | stat.S_IEXEC) + self.ns = os.path.join(self.t, "ns") + for n in ("demo-hp", "gone-cust", "operator"): + os.makedirs(os.path.join(self.ns, n)) + for f in ("ep0.secret", "local.secret", "fp"): + open(os.path.join(self.t, f), "w").write("x") + self.state = os.path.join(self.t, "state", "absent.json") + self.log = os.path.join(self.t, "calls.log") + self.env = dict(os.environ, PATH=bindir + ":" + os.environ["PATH"], EP0_COPY_NS_DIR=self.ns, + EP0_COPY_GC_STATE=self.state, EP0_TOKEN_FILE=os.path.join(self.t, "ep0.secret"), + EP0_FINGERPRINT_FILE=os.path.join(self.t, "fp"), LOCAL_TOKEN_FILE=os.path.join(self.t, "local.secret"), + FAKE_LOG=self.log, FAKE_EP0_NS=json.dumps(["demo-hp", "operator"])) + + def run_gc(self, *args, **env): + e = dict(self.env, **env) + return subprocess.run([SCRIPT] + list(args), env=e, capture_output=True, text=True) + + def deletes(self): + if not os.path.exists(self.log): + return [] + return [l.split()[2] for l in open(self.log) if l.startswith("namespace delete")] + + def seed(self, ns, first): + os.makedirs(os.path.dirname(self.state), exist_ok=True) + json.dump({ns: first}, open(self.state, "w")) + + def test_apply_deletes_after_grace(self): + self.seed("gone-cust", (dt.date.today() - dt.timedelta(days=8)).isoformat()) + r = self.run_gc("--apply") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(self.deletes(), ["gone-cust"]) + self.assertIn("pw=set", open(self.log).read()) + self.assertEqual(json.load(open(self.state)), {}) + + def test_inside_grace_nothing_deleted(self): + r = self.run_gc("--apply") + self.assertEqual(r.returncode, 0) + self.assertEqual(self.deletes(), []) + self.assertIn("gone-cust", json.load(open(self.state))) + + def test_dry_run_never_deletes(self): + self.seed("gone-cust", "2026-01-01") + r = self.run_gc() + self.assertEqual(self.deletes(), []) + self.assertIn("DRY RUN", r.stdout) + + def test_ep0_empty_list_aborts(self): + self.seed("gone-cust", "2026-01-01") + r = self.run_gc("--apply", FAKE_EP0_NS="[]") + self.assertEqual(r.returncode, 2) + self.assertEqual(self.deletes(), []) + + def test_ep0_unreachable_aborts(self): + self.seed("gone-cust", "2026-01-01") + r = self.run_gc("--apply", FAKE_EP0_FAIL="1") + self.assertEqual(r.returncode, 2) + self.assertEqual(self.deletes(), []) + self.assertEqual(json.load(open(self.state)), {"gone-cust": "2026-01-01"}) # state untouched + + def test_secret_never_printed(self): + open(os.path.join(self.t, "ep0.secret"), "w").write("SEKRIT-VALUE") + self.seed("gone-cust", "2026-01-01") + r = self.run_gc("--apply") + self.assertNotIn("SEKRIT", r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main()