From feea06062a4d36c0461888a0cc1f8a6e5e8d144a Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 29 Jun 2026 07:50:30 +0200 Subject: [PATCH] secrets: rotate + de-git Resend key; source from out-of-band Secret/resend-api Resend send-scoped key was committed in plaintext in manifests/hub.yaml (ConfigMap) and manifests/felhom.secret.yaml. Rotated to a new key and removed from git. - hub: new RESEND_API_KEY env override (cmd/hub/main.go), mirrors REGISTRY_TOKEN; ConfigMap resend_api_key now an empty placeholder; Deployment injects from Secret/resend-api. Image 0.17.0. - contact-mailer: secretKeyRef repointed contact-mailer-config -> resend-api. - felhom.secret.yaml: contact-mailer-config Secret removed; healthchecks EMAIL_HOST_PASSWORD blanked (workload not deployed). - documentation/runbooks/secrets.md: out-of-band secret model + create/rotate steps. Secret/resend-api is created imperatively out-of-band and is NOT committed. No secret value appears in this repo. Co-Authored-By: Claude Opus 4.8 (1M context) --- documentation/runbooks/secrets.md | 75 +++++++++++++++++++++++++++++++ hub/CHANGELOG.md | 15 +++++++ hub/cmd/hub/main.go | 6 +++ manifests/contact-mailer.yaml | 12 ++--- manifests/felhom.secret.yaml | 16 +++---- manifests/hub.yaml | 14 +++++- 6 files changed, 122 insertions(+), 16 deletions(-) create mode 100644 documentation/runbooks/secrets.md diff --git a/documentation/runbooks/secrets.md b/documentation/runbooks/secrets.md new file mode 100644 index 0000000..9292ed1 --- /dev/null +++ b/documentation/runbooks/secrets.md @@ -0,0 +1,75 @@ +# Runbook — out-of-band secrets (felhom-system) + +**Model:** out-of-band `kubectl` + this runbook (DECISION model **A**). Secret *values* are **never** +committed to git. The manifests in `manifests/` carry only **placeholders + comments** that point here. +The live values live in the operator's out-of-band store (password manager / escrow) and in the cluster +as Secrets created imperatively with the commands below. + +> The `felhom` ArgoCD app syncs the **entire** `manifests/` dir, so any Secret committed there is +> re-applied on every sync. Secrets that must stay out of git are therefore created as objects that are +> **not present in `manifests/`** (ArgoCD does not manage them), e.g. `Secret/resend-api` below. + +--- + +## Resend API key — `Secret/resend-api` + +**What uses it (live consumers):** + +| Consumer | How it reads the key | +|----------|----------------------| +| `hub` (`Deployment/hub`) | env `RESEND_API_KEY` ← `secretKeyRef: resend-api/RESEND_API_KEY`; the hub binary's `RESEND_API_KEY` env override fills `notifications.resend_api_key` (the committed ConfigMap field is an empty placeholder). | +| `contact-mailer` (`Deployment/contact-mailer`) | env `RESEND_API_KEY` ← `secretKeyRef: resend-api/RESEND_API_KEY`. | +| `healthchecks` (**not currently deployed**) | when deployed, wire `EMAIL_HOST_PASSWORD` to `secretKeyRef: resend-api/RESEND_API_KEY`; do **not** inline the value into `felhom.secret.yaml`. | +| Gmail "Send mail as" (`info@` / `admin@felhom.eu`) | external — the key is the SMTP password; updated manually in Gmail settings, **not** in the cluster. | + +**Where the value lives out-of-band:** the operator's password manager, entry "Felhom Resend API key +(app-relay + alerts)". Send-scoped. **Never** written to a file on a host, the guest, or any commit. + +### Create / rotate the Secret + +The key is available as the `RESEND_API` env var in `kisfenyo`'s shell on the build host +(192.168.0.180). It is exported from `~/.bashrc`, so it is present in an **interactive** shell +(`bash -ic`) but **not** in a plain non-interactive `ssh host 'cmd'`. + +Create (or update) the Secret without ever echoing the value — render the Secret YAML in the +(non-sudo) interactive shell where `$RESEND_API` is set, and pipe it into `sudo kubectl apply`: + +```bash +# on 192.168.0.180, as kisfenyo: +bash -ic 'kubectl create secret generic resend-api -n felhom-system \ + --from-literal=RESEND_API_KEY="$RESEND_API" \ + --dry-run=client -o yaml' | sudo kubectl apply -f - +``` + +Then roll the consumers so they pick up the new value: + +```bash +sudo kubectl -n felhom-system rollout restart deploy/hub deploy/contact-mailer +sudo kubectl -n felhom-system rollout status deploy/hub deploy/contact-mailer --timeout=120s +``` + +### Verify a consumer can send + +- **contact-mailer:** submit the website contact form at (or, if `DEBUG=true`, + `curl -X POST https://felhom.eu/api/debug/test`) → mail arrives at `info@felhom.eu`. +- **hub:** trigger a notification (or the dispatcher test path) → mail arrives. +- **Gmail:** send a test from `info@felhom.eu` → delivers. + +### Rotating away from a compromised key (ordered — load-bearing) + +1. Create the **new** send-scoped key in Resend; store it out-of-band; set `RESEND_API` on host 180. +2. Run the create-Secret + rollout above with the new value. +3. Update Gmail "Send mail as" SMTP password to the new key. +4. **Verify every consumer sends on the new key** (above) — do this **before** step 5. +5. Only then: **delete the old key** in Resend. Re-verify one consumer still sends afterwards. + +> Never delete the old key before step 4 passes — a premature delete breaks live mail. + +--- + +## Other committed secrets (tracked, NOT yet de-gitted — backlog) + +`manifests/felhom.secret.yaml` still commits other plaintext secrets (`healthchecks-config` `SECRET_KEY` +/ `SUPERUSER_PASSWORD`, `umami-config`, `gitea-creds`). These are **out of scope** for the Resend +rotation but are the same hygiene problem; de-git them the same way (out-of-band `Secret/...` + +placeholder) when touched. Tracked here so the gap is visible. diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 47dc5ba..5cd9de0 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,20 @@ # Felhom Hub — Changelog +## v0.17.0 — Resend key sourced from a Secret, out of git (2026-06-29) + +Resend rotation + de-git hygiene. The hub's Resend API key was committed in plaintext in +`manifests/hub.yaml`'s `hub-config` ConfigMap; it is now sourced from an out-of-band Kubernetes +Secret (`resend-api`) and never lives in git. + +- **`cmd/hub/main.go`:** new `RESEND_API_KEY` env override for `notifications.resend_api_key`, mirroring + the existing `REGISTRY_TOKEN` / `DEFAULT_MIN_CONTROLLER_VERSION` k8s-Secret override pattern. When set, + it wins over the (now empty) ConfigMap field. No behaviour change when unset. +- The committed ConfigMap `resend_api_key` is now an empty placeholder pointing at + `documentation/runbooks/secrets.md`; the live value is injected from `Secret/resend-api` via env. +- Part of the Resend key rotation: the previously-exposed send-scoped key was rotated; the new key lives + only in the out-of-band store and `Secret/resend-api` (created imperatively, never committed). See the + secrets runbook. No secret value appears in this repo. + ## v0.16.0 — Day-0 artifact manifest (hub-vouched agent + golden checksums) (2026-06-28) The hub now serves a passphrase-authed **artifact manifest** so the host-bootstrap script can diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index b7d0bcc..0fe24f4 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -93,6 +93,12 @@ func main() { if v := os.Getenv("DEFAULT_MIN_CONTROLLER_VERSION"); v != "" { cfg.ControllerUpdates.DefaultMinVersion = v } + // Resend API key is sourced from Secret/resend-api (env RESEND_API_KEY), never from the + // committed ConfigMap — see documentation/runbooks/secrets.md. The ConfigMap field stays + // empty as a placeholder; this override is the live source. + if v := os.Getenv("RESEND_API_KEY"); v != "" { + cfg.Notifications.ResendAPIKey = v + } // Ensure data dir exists os.MkdirAll(cfg.Server.DataDir, 0755) diff --git a/manifests/contact-mailer.yaml b/manifests/contact-mailer.yaml index d956974..8b2ef7d 100644 --- a/manifests/contact-mailer.yaml +++ b/manifests/contact-mailer.yaml @@ -10,10 +10,10 @@ # # Option B: Import directly into k3s (single node): # # docker save contact-mailer:latest | sudo k3s ctr images import - # -# 2. Create the Secret with your Resend API key: -# kubectl create secret generic contact-mailer-config \ -# --namespace=felhom-system \ -# --from-literal=RESEND_API_KEY='re_xxxxxxxxxxxx' +# 2. The Resend API key comes from the out-of-band Secret/resend-api (NOT committed). +# Create it per documentation/runbooks/secrets.md (key sourced from the out-of-band store): +# kubectl create secret generic resend-api -n felhom-system \ +# --from-literal=RESEND_API_KEY="$RESEND_API" # # 3. Apply this manifest: # kubectl apply -f contact-mailer.yaml @@ -56,10 +56,12 @@ spec: ports: - containerPort: 8080 env: + # Resend API key — injected from the shared out-of-band Secret/resend-api (NOT committed). + # See documentation/runbooks/secrets.md. - name: RESEND_API_KEY valueFrom: secretKeyRef: - name: contact-mailer-config + name: resend-api key: RESEND_API_KEY - name: FROM_EMAIL value: "Felhom.eu " diff --git a/manifests/felhom.secret.yaml b/manifests/felhom.secret.yaml index 7ec801d..2746fe7 100644 --- a/manifests/felhom.secret.yaml +++ b/manifests/felhom.secret.yaml @@ -17,19 +17,17 @@ stringData: EMAIL_HOST: "smtp.resend.com" EMAIL_PORT: "587" EMAIL_HOST_USER: "resend" - EMAIL_HOST_PASSWORD: "re_XZZenCJs_LyJnU12jZWfEn9rK85Gc83DK" + # Resend API key — NOT committed. Healthchecks is not currently deployed; when it is, wire its + # EMAIL_HOST_PASSWORD to the out-of-band Secret/resend-api (key RESEND_API_KEY) via secretKeyRef + # instead of inlining a value here. See documentation/runbooks/secrets.md. + EMAIL_HOST_PASSWORD: "" EMAIL_USE_TLS: "True" EMAIL_USE_VERIFICATION: "False" DEFAULT_FROM_EMAIL: "monitoring@felhom.eu" --- -apiVersion: v1 -kind: Secret -metadata: - name: contact-mailer-config - namespace: felhom-system -type: Opaque -stringData: - RESEND_API_KEY: 're_XZZenCJs_LyJnU12jZWfEn9rK85Gc83DK' +# NOTE: the Resend API key (formerly Secret/contact-mailer-config RESEND_API_KEY) is no longer +# committed. It lives in the out-of-band Secret/resend-api (key RESEND_API_KEY), created imperatively +# per documentation/runbooks/secrets.md. Both the hub and contact-mailer now read from resend-api. --- apiVersion: v1 kind: Secret diff --git a/manifests/hub.yaml b/manifests/hub.yaml index 05d98b7..f1e1108 100644 --- a/manifests/hub.yaml +++ b/manifests/hub.yaml @@ -81,7 +81,10 @@ data: alerting: stale_threshold: "30m" notifications: - resend_api_key: "re_XZZenCJs_LyJnU12jZWfEn9rK85Gc83DK" + # Resend API key is NOT stored here. It is injected at runtime from Secret/resend-api + # via the RESEND_API_KEY env var (see Deployment below). The Secret is created out-of-band + # and is NOT committed — see documentation/runbooks/secrets.md. Leave this empty. + resend_api_key: "" registry: image: "gitea.dooplex.hu/admin/felhom-controller" # username + token injected via REGISTRY_USERNAME / REGISTRY_TOKEN env vars @@ -117,7 +120,7 @@ spec: spec: containers: - name: hub - image: gitea.dooplex.hu/admin/felhom-hub:0.16.0 + image: gitea.dooplex.hu/admin/felhom-hub:0.17.0 ports: - containerPort: 8080 name: http @@ -128,6 +131,13 @@ spec: # this auto-updates to it (unless a per-customer override is set via the operator UI). - name: DEFAULT_MIN_CONTROLLER_VERSION value: "0.87.0" + # Resend API key — injected from the out-of-band Secret/resend-api (NOT committed). + # See documentation/runbooks/secrets.md. Overrides the empty ConfigMap placeholder. + - name: RESEND_API_KEY + valueFrom: + secretKeyRef: + name: resend-api + key: RESEND_API_KEY - name: REGISTRY_USERNAME valueFrom: secretKeyRef: