From ce6a56691eaa87328ff93885b6cb21f55ed7d290 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Fri, 10 Jul 2026 20:49:48 +0200 Subject: [PATCH] =?UTF-8?q?hub=20v0.44.0:=20PBS=20DR=20tier=20SLICE=201=20?= =?UTF-8?q?=E2=80=94=20felhom-tenantsync=20surface=20(script+client)=20+?= =?UTF-8?q?=20hub=20provisioning=20flow=20(consume-once=20host=20secret,?= =?UTF-8?q?=20pbs=5Fdr=20desired-state=20descriptor,=20fail-closed=20+=20i?= =?UTF-8?q?dempotent,=20re-issue)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- REUSE.md | 10 + hub/CHANGELOG.md | 40 +++ hub/cmd/hub/main.go | 23 ++ hub/internal/api/handler.go | 4 + hub/internal/api/pbsdr.go | 47 +++ hub/internal/api/pbsdr_test.go | 80 +++++ hub/internal/store/pbsdr.go | 40 +++ hub/internal/store/pbsdr_test.go | 43 +++ hub/internal/store/store.go | 17 + hub/internal/tenantsync/client.go | 219 ++++++++++++ hub/internal/tenantsync/client_test.go | 300 ++++++++++++++++ hub/internal/web/configs.go | 21 ++ hub/internal/web/configs_debug_test.go | 1 + hub/internal/web/pbsdr.go | 308 ++++++++++++++++ hub/internal/web/pbsdr_test.go | 373 ++++++++++++++++++++ hub/internal/web/server.go | 9 + hub/internal/web/templates/config_form.html | 30 ++ manifests/hub.yaml | 16 + scripts/felhom-tenantsync.sh | 162 +++++++++ 19 files changed, 1743 insertions(+) create mode 100644 hub/internal/api/pbsdr.go create mode 100644 hub/internal/api/pbsdr_test.go create mode 100644 hub/internal/store/pbsdr.go create mode 100644 hub/internal/store/pbsdr_test.go create mode 100644 hub/internal/tenantsync/client.go create mode 100644 hub/internal/tenantsync/client_test.go create mode 100644 hub/internal/web/pbsdr.go create mode 100644 hub/internal/web/pbsdr_test.go create mode 100644 scripts/felhom-tenantsync.sh diff --git a/REUSE.md b/REUSE.md index 20868d9..3a130cd 100644 --- a/REUSE.md +++ b/REUSE.md @@ -66,6 +66,14 @@ | `(*Handler).handleArtifactManifest` | hub/internal/api/handler.go (~L1550) | `GET /api/v1/artifacts/{id}` | Serving the vouched set to the bootstrap script | Auth mirrors handleConfigRetrieve exactly. Unset manifest = empty fields, not an error. | | `normalizeFloorInput` / `normalizeSHA256` | hub/internal/web/configs.go (~L27 / ~L627) | `(raw string) (string, bool)` | Validating operator-typed versions / shas | Empty string is VALID (means "clear"). Reuse for any new version/sha form field. | +### PBS DR tier / tenantsync (hub/internal/tenantsync/, web/pbsdr.go, api/pbsdr.go) + +| Symbol | File | Short signature | Use for | Gotchas | +|---|---|---|---|---| +| `tenantsync.Client` (`Provision`/`Reissue`/`Fingerprint`) | hub/internal/tenantsync/client.go | `(ctx, customerID) (*Result, error)` | ep0 per-customer PBS tenancy over the pinned-SSH forced-command channel (the wgsync twin) | `Result.TokenSecret` is transient custody → `SaveHostPBSSecret` immediately, never log the struct. Error paths NEVER embed stdout (the secret channel) — do not "improve" diagnostics by quoting the response. `ErrTokenExists` is typed: provision refuses an existing token; re-issue is the explicit path. | +| `(*Store).SaveHostPBSSecret` / `ConsumeHostPBSSecret` | hub/internal/store/pbsdr.go | `(hostID, value)` / `(hostID) (string, error)` | HOST-scoped consume-once secret (the one_time_secrets host twin) | Same-tx mark-consumed; re-save resets consumption (re-issue supersedes). The agent consumes via `POST /api/v1/hosts/{id}/pbs/consume-token` (api/pbsdr.go). | +| `(*Server).applyPBSDR` + `mergePBSDR`/`readPBSDR` | hub/internal/web/pbsdr.go | `(ctx, r, cfg) error` | The config form's PBS DR section → HOST desired_json `pbs_dr` descriptor + generation bump | Descriptor lives in the host desired_json, NOT ConfigJSON (buildConfigJSON drops foreign keys on re-save). Fail-closed before SaveCustomerConfig; already-provisioned = success-no-op (red-proofed); disable keeps the ep0 tenancy. | + ### Config generation & secrets hygiene (hub/internal/configgen/) | Symbol | File | Short signature | Use for | Gotchas | @@ -129,6 +137,8 @@ | `api.ConfigTemplateProvider` | hub/internal/api/handler.go (~L24) | `web.TemplateFetcher` (Gitea-pulled controller.yaml template) | stub providers in api tests | | `api.LatestVersionProvider` | hub/internal/api/handler.go (~L31) | `web.VersionChecker` (registry poll) | hub/internal/api/config_version_ack_test.go | | `mailRateLimiter.now` (func seam) | hub/internal/api/mail.go (~L27) | `time.Now` | hub/internal/api/mail_test.go clock injection | +| `web.tenancyProvisioner` | hub/internal/web/pbsdr.go | `*tenantsync.Client` (pinned SSH to ep0's felhom-tenantsync) | `fakeTenancy` in hub/internal/web/pbsdr_test.go; in-process SSH server in hub/internal/tenantsync/client_test.go | +| Cross-repo: ep0 tenancy surface | `scripts/felhom-tenantsync.sh` (JSON stdin/stdout forced command) | installed on ep0 per runbook offsite-endpoint.md §10 | provision/reissue/fingerprint ops; token secret rides stdout ONLY; the peersync script/key are untouched | | Cross-repo: controller → hub | `POST /api/v1/report` (frozen) + `POST /api/v1/event` | felhom-controller repo | new event types MUST enter `allowedEventTypes` (hub/internal/api/handler.go ~L1063) or the controller gets 400 | | Cross-repo: agent → hub | `POST /api/v1/host-report`, `/host-enroll`, jobs/desired-state/escrow routes (handler.go ~L145–189) | felhom-agent repo | hub/internal/api/host_test.go, desired_test.go, escrow_test.go, dr_test.go | | Cross-repo: Day-0 bootstrap → hub | `GET /api/v1/config/{id}` + `/artifacts/{id}` (X-Retrieval-Password) | scripts/felhom-host-install.sh (fetches + sha256-verifies against the hub-vouched manifest) | hub/internal/api/artifact_test.go | diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 4926f2e..79b4b7e 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,45 @@ # Felhom Hub — Changelog +## v0.44.0 — PBS DR tier SLICE 1: ep0 tenantsync surface + hub provisioning (2026-07-10) + +Builds on SPIKE-pbs-tier-provisioning (00afadc). The operator ticks "PBS DR tier (ep0)" on a +customer config → the hub verifies the host's WG peer (the agent self-registers it; absence is +fail-closed) → provisions the per-customer ep0 PBS namespace + privilege-separated token over the +NEW `felhom-tenantsync` forced-command surface (peersync untouched) → stores the token secret +CONSUME-ONCE, host-scoped → serves the non-secret descriptor via the host desired-state (generation +bump). The agent apply-bridge is SLICE 2 — nothing is live-provisioned yet. + +- **`scripts/felhom-tenantsync.sh` v1.0.0** (installed on ep0 per runbook §10): JSON-on-stdin/stdout; + ops `provision` (existing token = hard error `token_exists` — re-issue is explicit), `reissue` + (delete-token purges ACLs → recreate → re-grant), `fingerprint`. Dual-grant per spike §3; + own-namespace self-check with one regen retry (the spike's transient-403 note) then rollback. + Secret hygiene: the token secret rides stdout ONLY (all tool stdout → stderr; never a file/argv). + NO deprovision op — namespace/data deletion stays a deliberate, separate decision. +- **`internal/tenantsync`**: the wgsync twin — pinned host key (exact-match, constrained + HostKeyAlgorithms), per-op JSON exec, typed `ErrTokenExists`. Divergence from wgsync: error + messages NEVER embed stdout (the secret channel) — red-proof-style contract test + (`TestErrors_NeverEmbedStdout`). +- **Store**: `host_pbs_secrets` (host-scoped consume-once, the `one_time_secrets` twin) + + `SaveHostPBSSecret`/`ConsumeHostPBSSecret` (same-tx mark; re-save resets). +- **API**: `POST /api/v1/hosts/{id}/pbs/consume-token` — per-host key, self-scoped (global key = + operator recovery); 200 exactly once → 404; a foreign key's 403 does NOT burn the secret. + (Task spec wrote `/host/{id}/…`; implemented under `/hosts/` for namespace consistency with + every other agent-facing route.) +- **Web**: config-form section "PBS DR tier (ep0)" (enable + storage-id, default `felhom-pbs` — + the descriptor carries the id so the slice-2 bridge is name-agnostic and the demo's + `felhom-offsite` adoption dissolves the naming collision) + provisioned line + **Re-issue PBS + credentials** (the offsite F4 precedent). `applyPBSDR` merges the non-secret `pbs_dr` descriptor + into the HOST `desired_json` (admin-set path, `SetHostDesired` bump) — ConfigJSON never carries + it. Fail-closed on: no tenantsync key, no enrolled host, no WG peer, no endpoint record, + tenantsync error, `token_exists` (message points at Re-issue). Already-provisioned re-save = + success-no-op (no re-key, no second secret, no spurious bump). Disable = descriptor + `enabled:false`, tenancy kept. +- **Deploy**: `manifests/hub.yaml` gains `TENANTSYNC_SSH_KEY_FILE` + optional `Secret/tenantsync` + mount (same endpoint addr + pinned host key as peersync, its own key). +- Red-proofs run and recorded (REPORT.md): consume-once mark drop → the secret re-serves (store + + API layers); fail-closed guard swallow → 303 half-save; idempotency short-circuit drop → + token rotation + fresh secret + spurious bump. + ## v0.43.1 — Git Sync form hint: credentials are optional (2026-07-10) Pairs with controller v0.112.0 (anonymous registry self-update). The config editor's Git Sync section diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index f98b7f8..65be4d9 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -23,6 +23,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/web" + "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" "gitea.dooplex.hu/admin/felhom-hub/internal/wgsync" "gopkg.in/yaml.v3" ) @@ -366,6 +367,28 @@ func main() { } else { logger.Printf("[INFO] WG peer-sync disabled (endpoint not configured)") } + + // PBS DR tier (SLICE 1): the tenantsync client — same endpoint address + pinned host key + // as peersync, its OWN private key (the authorized_keys line selects the forced command). + tsKeyFile := os.Getenv("TENANTSYNC_SSH_KEY_FILE") + if wgAddr != "" && wgHostKey != "" && tsKeyFile != "" { + keyPEM, err := os.ReadFile(tsKeyFile) + if os.IsNotExist(err) { + // The env names the mount path unconditionally; the optional Secret may not exist yet. + logger.Printf("[INFO] PBS DR tenantsync disabled (key %s not present)", tsKeyFile) + } else if err != nil { + logger.Printf("[ERROR] PBS DR tenantsync disabled: read key file %s: %v", tsKeyFile, err) + } else if tsClient, err := tenantsync.New(tenantsync.Config{ + Addr: wgAddr, User: wgUser, PrivateKey: keyPEM, HostKeyLine: wgHostKey, + }, logger); err != nil { + logger.Printf("[ERROR] PBS DR tenantsync disabled: %v", err) + } else { + webServer.SetTenantSync(tsClient) + logger.Printf("[INFO] PBS DR tenantsync enabled (endpoint %s, user %s)", wgAddr, wgUser) + } + } else { + logger.Printf("[INFO] PBS DR tenantsync disabled (key or endpoint not configured)") + } } // Session cleanup — removes expired sessions every hour diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 3f8ba71..5d38225 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -181,6 +181,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/wg"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/wg") h.handleRegisterHostWG(w, r, hostID) + // PBS DR tier (SLICE 1): the agent's consume-once fetch of its PBS token secret (api/pbsdr.go). + case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/pbs/consume-token"): + hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/pbs/consume-token") + h.handleConsumePBSToken(w, r, hostID) // Desired-state serving (slice 10A) — per-host-key, self-scoped (a host reads only its own). case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/desired-state"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/desired-state") diff --git a/hub/internal/api/pbsdr.go b/hub/internal/api/pbsdr.go new file mode 100644 index 0000000..13f4f0f --- /dev/null +++ b/hub/internal/api/pbsdr.go @@ -0,0 +1,47 @@ +package api + +// PBS DR tier (SLICE 1): the agent-facing consume-once endpoint for the host's PBS token secret. +// The hub stored the secret at provision time (tenantsync → Store.SaveHostPBSSecret); the agent +// fetches it EXACTLY ONCE with its per-host key while applying the desired-state pbs_dr block +// (SLICE 2). The controller-side offsite consume endpoint is the precedent; this is its host-side +// twin. The secret value is never logged. + +import ( + "database/sql" + "encoding/json" + "net/http" +) + +// handleConsumePBSToken serves POST /api/v1/hosts/{host_id}/pbs/consume-token. Per-host key, +// self-scoped (the global key may consume on a host's behalf — the operator recovery path). +// 200 exactly once per stored secret; 404 when absent or already consumed; 403 on a foreign key. +func (h *Handler) handleConsumePBSToken(w http.ResponseWriter, r *http.Request, pathHostID string) { + authHostID, _, isGlobal, ok := h.checkAuthHost(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if pathHostID == "" { + http.Error(w, "Missing host_id", http.StatusBadRequest) + return + } + if !isGlobal && authHostID != pathHostID { + http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden) + return + } + + secret, err := h.store.ConsumeHostPBSSecret(pathHostID) + if err == sql.ErrNoRows { + http.Error(w, "No unconsumed PBS token secret for this host", http.StatusNotFound) + return + } + if err != nil { + h.logger.Printf("[ERROR] pbs consume-token %s: %v", pathHostID, err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + h.logger.Printf("[INFO] pbs token secret consumed by host %s (single-use; value withheld from logs)", pathHostID) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{"token_secret": secret}) +} diff --git a/hub/internal/api/pbsdr_test.go b/hub/internal/api/pbsdr_test.go new file mode 100644 index 0000000..e7ef0f5 --- /dev/null +++ b/hub/internal/api/pbsdr_test.go @@ -0,0 +1,80 @@ +package api + +// PBS DR SLICE 1 — the agent-facing consume-once endpoint. The contract: 200 with the secret +// EXACTLY once per stored value, 404 after (and when nothing is stored), 403 on a foreign +// host's key WITHOUT burning the secret, 401 unauthenticated. + +import ( + "encoding/json" + "net/http" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +func TestConsumePBSToken_OnceThen404(t *testing.T) { + h, st, _ := newTestHandler(t) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + if err := st.SaveHostPBSSecret("h1", "tok-secret-1"); err != nil { + t.Fatalf("seed secret: %v", err) + } + + rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "HKEY", "") + if rr.Code != http.StatusOK { + t.Fatalf("first consume = %d, want 200 (%s)", rr.Code, rr.Body.String()) + } + var resp map[string]string + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("response parse: %v", err) + } + if resp["token_secret"] != "tok-secret-1" { + t.Errorf("token_secret = %q, want tok-secret-1", resp["token_secret"]) + } + + // Consume-once: the second fetch MUST 404. (Red-proof: drop the consumed_at UPDATE in + // store.ConsumeHostPBSSecret → this returns 200 with the secret again → FAIL.) + rr = do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "HKEY", "") + if rr.Code != http.StatusNotFound { + t.Fatalf("second consume = %d (%s), want 404 — single-use broken", rr.Code, rr.Body.String()) + } +} + +func TestConsumePBSToken_ForeignKeyForbiddenAndSecretSurvives(t *testing.T) { + h, st, _ := newTestHandler(t) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c2", APIKey: "HKEY2"}) + if err := st.SaveHostPBSSecret("h1", "tok-secret-1"); err != nil { + t.Fatalf("seed secret: %v", err) + } + + // h2's key against h1's path → 403, and the attempt must NOT consume h1's secret. + rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "HKEY2", "") + if rr.Code != http.StatusForbidden { + t.Fatalf("foreign-key consume = %d, want 403", rr.Code) + } + rr = do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "HKEY", "") + if rr.Code != http.StatusOK { + t.Fatalf("own consume after foreign 403 = %d, want 200 — the 403 burned the secret", rr.Code) + } +} + +func TestConsumePBSToken_AuthMatrix(t *testing.T) { + h, st, _ := newTestHandler(t) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + st.SaveHostPBSSecret("h1", "tok-secret-1") + + if rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "", ""); rr.Code != http.StatusUnauthorized { + t.Errorf("unauthenticated = %d, want 401", rr.Code) + } + if rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "bogus", ""); rr.Code != http.StatusUnauthorized { + t.Errorf("bogus key = %d, want 401", rr.Code) + } + // The global key may consume on a host's behalf (operator recovery path). + if rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", globalKey, ""); rr.Code != http.StatusOK { + t.Errorf("global key = %d, want 200", rr.Code) + } + // Nothing stored (just consumed above) → 404, not an error leak. + if rr := do(h, http.MethodPost, "/hosts/h1/pbs/consume-token", "HKEY", ""); rr.Code != http.StatusNotFound { + t.Errorf("post-consume = %d, want 404", rr.Code) + } +} diff --git a/hub/internal/store/pbsdr.go b/hub/internal/store/pbsdr.go new file mode 100644 index 0000000..99e4642 --- /dev/null +++ b/hub/internal/store/pbsdr.go @@ -0,0 +1,40 @@ +package store + +// PBS DR tier (SLICE 1): the HOST-scoped one-time PBS token secret — the host/agent twin of the +// customer-scoped one_time_secrets pair (SaveOneTimeSecret/ConsumeOneTimeSecret). The hub stores +// the tenantsync-returned token secret here; the agent consumes it EXACTLY ONCE with its per-host +// key. Transient custody: never logged, never in desired-state or any served config. + +// SaveHostPBSSecret stores (last-write-wins) the one-time PBS token secret for a host, resetting +// the consumed flag (a re-issue supersedes any prior unconsumed value). Never logged. +func (s *Store) SaveHostPBSSecret(hostID, value string) error { + _, err := s.db.Exec(` + INSERT INTO host_pbs_secrets (host_id, value, created_at, consumed_at) + VALUES (?, ?, datetime('now'), NULL) + ON CONFLICT(host_id) DO UPDATE SET value = excluded.value, created_at = datetime('now'), consumed_at = NULL`, + hostID, value) + return err +} + +// ConsumeHostPBSSecret returns the host's one-time PBS token secret and marks it consumed in the +// SAME transaction (single use). A second call — or a call when none is stored — returns +// ("", sql.ErrNoRows). The value is never logged. +func (s *Store) ConsumeHostPBSSecret(hostID string) (string, error) { + tx, err := s.db.Begin() + if err != nil { + return "", err + } + defer tx.Rollback() + var value string + err = tx.QueryRow(`SELECT value FROM host_pbs_secrets WHERE host_id = ? AND consumed_at IS NULL`, hostID).Scan(&value) + if err != nil { + return "", err // sql.ErrNoRows when absent OR already consumed + } + if _, err := tx.Exec(`UPDATE host_pbs_secrets SET consumed_at = datetime('now') WHERE host_id = ?`, hostID); err != nil { + return "", err + } + if err := tx.Commit(); err != nil { + return "", err + } + return value, nil +} diff --git a/hub/internal/store/pbsdr_test.go b/hub/internal/store/pbsdr_test.go new file mode 100644 index 0000000..30f1ad1 --- /dev/null +++ b/hub/internal/store/pbsdr_test.go @@ -0,0 +1,43 @@ +package store + +import ( + "database/sql" + "testing" +) + +// The host-scoped consume-once contract (PBS DR SLICE 1): exactly one read per stored value, +// a re-save resets the consumed flag (re-issue supersedes), absence is sql.ErrNoRows. +func TestHostPBSSecret_ConsumeOnce(t *testing.T) { + s := newTestStore(t) + + if _, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows { + t.Fatalf("consume with nothing stored = %v, want sql.ErrNoRows", err) + } + + if err := s.SaveHostPBSSecret("h1", "secret-1"); err != nil { + t.Fatalf("save: %v", err) + } + got, err := s.ConsumeHostPBSSecret("h1") + if err != nil || got != "secret-1" { + t.Fatalf("first consume = (%q, %v), want (secret-1, nil)", got, err) + } + // Single use: the second consume MUST be ErrNoRows (red-proof: dropping the consumed_at + // UPDATE in ConsumeHostPBSSecret makes this assert fail with the value returned again). + if got, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows { + t.Fatalf("second consume = (%q, %v), want sql.ErrNoRows — consume-once broken", got, err) + } + + // Re-issue path: a fresh save resets consumption and serves the NEW value once. + if err := s.SaveHostPBSSecret("h1", "secret-2"); err != nil { + t.Fatalf("re-save: %v", err) + } + got, err = s.ConsumeHostPBSSecret("h1") + if err != nil || got != "secret-2" { + t.Fatalf("post-reissue consume = (%q, %v), want (secret-2, nil)", got, err) + } + + // Host isolation: h2 never sees h1's rows. + if _, err := s.ConsumeHostPBSSecret("h2"); err != sql.ErrNoRows { + t.Fatalf("foreign host consume = %v, want sql.ErrNoRows", err) + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 70a67ba..f4f195c 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -440,6 +440,23 @@ func (s *Store) migrate() error { return err } + // PBS DR tier (SLICE 1, v0.44.0): the one-time PBS token secret, HOST-scoped — the sibling of + // one_time_secrets (customer/controller custody) for host/agent custody. The hub receives the + // secret over the tenantsync channel, stores it here, and the AGENT consumes it exactly once + // (POST /api/v1/hosts/{id}/pbs/consume-token, per-host key). consumed_at marks it spent; a + // re-issue supersedes any unconsumed value. Never logged, never in desired-state/ConfigJSON. + _, err = s.db.Exec(` + CREATE TABLE IF NOT EXISTS host_pbs_secrets ( + host_id TEXT PRIMARY KEY, + value TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + consumed_at DATETIME + ); + `) + if err != nil { + return err + } + // v0.43.0 — remote app-log diagnostics. Additive columns on app_log_issues: // context = JSON array of ±5 redacted lines around the FIRST occurrence (first capture // wins — stable repro context, no churn); context_customer = whose box it came from diff --git a/hub/internal/tenantsync/client.go b/hub/internal/tenantsync/client.go new file mode 100644 index 0000000..f429444 --- /dev/null +++ b/hub/internal/tenantsync/client.go @@ -0,0 +1,219 @@ +// Package tenantsync drives the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier +// SLICE 1) — the structural twin of internal/wgsync: SSH with a PINNED host key (exact-match or +// refuse, no fallback) to a forced-command script (`felhom-tenantsync` — its OWN key + sudoers +// line; the peersync surface is untouched). JSON on stdin, JSON on stdout, one op per session. +// +// SECRET HYGIENE (load-bearing divergence from wgsync): the script's stdout carries the one-time +// PBS token secret. It is parsed into Result and handed to the caller for the consume-once store +// write — it is NEVER logged, and error messages NEVER embed stdout bytes (stderr only). Do not +// "improve" the diagnostics by quoting the response. +package tenantsync + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "regexp" + "strings" + "time" + + "golang.org/x/crypto/ssh" +) + +// ErrTokenExists is the typed "provision refused: the token already exists" outcome — the hub +// treats it as a state mismatch (a descriptor should exist; re-issue is the explicit recovery). +var ErrTokenExists = errors.New("tenantsync: token already exists on the endpoint (re-issue is the explicit path)") + +// customerIDRe mirrors the script's validation — refuse client-side before a wasted SSH round-trip. +var customerIDRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}$`) + +// Config configures the SSH client. Addr/User/HostKeyLine are typically the SAME values as the +// peersync client (same box, same low-priv user, same pinned host key); PrivateKey is tenantsync's +// OWN key (the authorized_keys line selects the forced command). +type Config struct { + Addr string // "host:22" + User string // "felhom-peersync" + PrivateKey []byte // PEM private key (from the mounted Secret file) + HostKeyLine string // single authorized_keys-format line of the endpoint's host pubkey + Timeout time.Duration // default 60s (tenancy ops run several PBS commands) +} + +// Result is the script's ok-response: the descriptor fields + the ONE-TIME token secret. +// TokenSecret is transient custody — store it consume-once immediately, never log the struct. +type Result struct { + TokenID string `json:"token_id"` + TokenSecret string `json:"token_secret"` + Fingerprint string `json:"fingerprint"` + Datastore string `json:"datastore"` + Namespace string `json:"namespace"` +} + +// Client is a pinned-host-key SSH per-op executor. Construct with New (parses keys up front). +type Client struct { + addr string + user string + signer ssh.Signer + hostKey ssh.PublicKey + timeout time.Duration + logger *log.Logger +} + +// New builds a Client, failing early on an unparsable private key or host-key line. +func New(cfg Config, logger *log.Logger) (*Client, error) { + if cfg.Addr == "" || cfg.User == "" { + return nil, fmt.Errorf("tenantsync: Addr and User are required") + } + signer, err := ssh.ParsePrivateKey(cfg.PrivateKey) + if err != nil { + return nil, fmt.Errorf("tenantsync: parse private key: %w", err) + } + hostKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(cfg.HostKeyLine)) + if err != nil { + return nil, fmt.Errorf("tenantsync: parse host key line: %w", err) + } + timeout := cfg.Timeout + if timeout == 0 { + timeout = 60 * time.Second + } + if logger == nil { + logger = log.Default() + } + return &Client{ + addr: cfg.Addr, user: cfg.User, signer: signer, + hostKey: hostKey, timeout: timeout, logger: logger, + }, nil +} + +// Provision creates the customer's namespace + privilege-separated token on the endpoint. +// An already-existing token is the typed ErrTokenExists (never silently re-keyed). +func (c *Client) Provision(ctx context.Context, customerID string) (*Result, error) { + return c.tenancyOp(ctx, "provision", customerID) +} + +// Reissue explicitly re-keys the customer's token (delete + recreate + re-grant, script-side). +func (c *Client) Reissue(ctx context.Context, customerID string) (*Result, error) { + return c.tenancyOp(ctx, "reissue", customerID) +} + +// Fingerprint returns the endpoint PBS's API cert fingerprint (the descriptor field). +func (c *Client) Fingerprint(ctx context.Context) (string, error) { + stdout, stderr, runErr := c.exec(ctx, []byte(`{"op":"fingerprint"}`)) + resp, err := parseResponse(stdout, stderr, runErr) + if err != nil { + return "", err + } + if resp.Fingerprint == "" { + return "", fmt.Errorf("tenantsync: fingerprint op returned an empty fingerprint") + } + return resp.Fingerprint, nil +} + +func (c *Client) tenancyOp(ctx context.Context, op, customerID string) (*Result, error) { + if !customerIDRe.MatchString(customerID) { + return nil, fmt.Errorf("tenantsync: invalid customer_id %q", customerID) + } + payload, err := json.Marshal(map[string]string{"op": op, "customer_id": customerID}) + if err != nil { + return nil, err + } + stdout, stderr, runErr := c.exec(ctx, payload) + resp, err := parseResponse(stdout, stderr, runErr) + if err != nil { + return nil, err + } + if resp.TokenSecret == "" || resp.TokenID == "" || resp.Namespace == "" || resp.Fingerprint == "" || resp.Datastore == "" { + // Field NAMES only — never values (TokenSecret). + return nil, fmt.Errorf("tenantsync: %s response is missing required fields", op) + } + c.logger.Printf("[INFO] tenantsync: %s ok for %s (ns=%s, token_id=%s; secret withheld from logs)", + op, customerID, resp.Namespace, resp.TokenID) + return &resp.Result, nil +} + +// response is the script's stdout contract — ok carries the Result fields, error carries code+error. +type response struct { + Status string `json:"status"` + Code string `json:"code"` + Error string `json:"error"` + Result +} + +// parseResponse turns (stdout, stderr, runErr) into a typed outcome. The script emits its error +// JSON on stdout and exits 1, so a run error is parsed for the typed code FIRST; only when stdout +// carries no usable JSON does the raw failure (with stderr, NEVER stdout) surface. +func parseResponse(stdout, stderr []byte, runErr error) (*response, error) { + var resp response + parseOK := json.Unmarshal(bytes.TrimSpace(stdout), &resp) == nil + if parseOK && resp.Status == "error" { + if resp.Code == "token_exists" { + return nil, ErrTokenExists + } + return nil, fmt.Errorf("tenantsync: endpoint refused: %s (code %s)", resp.Error, resp.Code) + } + if runErr != nil { + return nil, fmt.Errorf("tenantsync: remote op failed: %w (stderr: %s)", + runErr, strings.TrimSpace(string(stderr))) + } + if !parseOK || resp.Status != "ok" { + // stdout may carry the secret — report shape only, never bytes. + return nil, fmt.Errorf("tenantsync: malformed endpoint response (%d stdout bytes; stderr: %s)", + len(bytes.TrimSpace(stdout)), strings.TrimSpace(string(stderr))) + } + return &resp, nil +} + +// exec runs one forced-command session: payload on stdin, returns stdout/stderr. The connection +// discipline (pinned key, constrained HostKeyAlgorithms, deadline both sides of the handshake) +// is wgsync.Push's, verbatim — that shape is live-proven against the real sshd. +func (c *Client) exec(ctx context.Context, payload []byte) (stdout, stderr []byte, err error) { + sshCfg := &ssh.ClientConfig{ + User: c.user, + Auth: []ssh.AuthMethod{ssh.PublicKeys(c.signer)}, + HostKeyCallback: ssh.FixedHostKey(c.hostKey), + // Constrain negotiation to the PINNED key's algorithm — a multi-hostkey sshd (stock: + // ECDSA + ed25519) otherwise presents a different type and FixedHostKey refuses a + // legitimate server (wgsync S1 live finding). + HostKeyAlgorithms: []string{c.hostKey.Type()}, + Timeout: c.timeout, + } + dialer := net.Dialer{Timeout: c.timeout} + conn, err := dialer.DialContext(ctx, "tcp", c.addr) + if err != nil { + return nil, nil, fmt.Errorf("tenantsync: dial %s: %w", c.addr, err) + } + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } else { + conn.SetDeadline(time.Now().Add(c.timeout)) + } + sconn, chans, reqs, err := ssh.NewClientConn(conn, c.addr, sshCfg) + if err != nil { + conn.Close() + return nil, nil, fmt.Errorf("tenantsync: ssh handshake %s: %w", c.addr, err) + } + client := ssh.NewClient(sconn, chans, reqs) + defer client.Close() + conn.SetDeadline(time.Time{}) + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + + session, err := client.NewSession() + if err != nil { + return nil, nil, fmt.Errorf("tenantsync: session: %w", err) + } + defer session.Close() + + var outBuf, errBuf bytes.Buffer + session.Stdin = bytes.NewReader(payload) + session.Stdout = &outBuf + session.Stderr = &errBuf + + // The forced command overrides this string; it documents intent on the wire. + runErr := session.Run("felhom-tenantsync") + return outBuf.Bytes(), errBuf.Bytes(), runErr +} diff --git a/hub/internal/tenantsync/client_test.go b/hub/internal/tenantsync/client_test.go new file mode 100644 index 0000000..98892b4 --- /dev/null +++ b/hub/internal/tenantsync/client_test.go @@ -0,0 +1,300 @@ +package tenantsync + +// The tenancy SSH client against an IN-PROCESS x/crypto/ssh server (the wgsync client_test +// pattern). Beyond the transport round-trip, these pin the package's two load-bearing contracts: +// the typed token_exists outcome, and the secret-hygiene rule that NO error message ever embeds +// stdout bytes (stdout is the secret channel). + +import ( + "bytes" + "context" + "encoding/pem" + "errors" + "io" + "log" + "net" + "strings" + "sync" + "testing" + "time" + + "crypto/ed25519" + "crypto/rand" + + "golang.org/x/crypto/ssh" +) + +func testKeys(t *testing.T) ([]byte, ssh.Signer) { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("ed25519: %v", err) + } + block, err := ssh.MarshalPrivateKey(priv, "") + if err != nil { + t.Fatalf("MarshalPrivateKey: %v", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("NewSignerFromKey: %v", err) + } + return pem.EncodeToMemory(block), signer +} + +type testServer struct { + addr string + mu sync.Mutex + captured []byte + cmd string +} + +func startTestServer(t *testing.T, hostSigner, clientSigner ssh.Signer, + stdoutResp, stderrResp string, exitStatus uint32) *testServer { + t.Helper() + cfg := &ssh.ServerConfig{ + PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + if bytes.Equal(key.Marshal(), clientSigner.PublicKey().Marshal()) { + return &ssh.Permissions{}, nil + } + return nil, io.EOF + }, + } + cfg.AddHostKey(hostSigner) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { ln.Close() }) + srv := &testServer{addr: ln.Addr().String()} + + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg) + if err != nil { + return + } + defer sconn.Close() + go ssh.DiscardRequests(reqs) + for newCh := range chans { + if newCh.ChannelType() != "session" { + newCh.Reject(ssh.UnknownChannelType, "unsupported") + continue + } + ch, chReqs, err := newCh.Accept() + if err != nil { + continue + } + go func() { + for req := range chReqs { + if req.Type == "exec" { + var p struct{ Command string } + ssh.Unmarshal(req.Payload, &p) + srv.mu.Lock() + srv.cmd = p.Command + srv.mu.Unlock() + req.Reply(true, nil) + data, _ := io.ReadAll(ch) + srv.mu.Lock() + srv.captured = data + srv.mu.Unlock() + if stderrResp != "" { + ch.Stderr().Write([]byte(stderrResp)) + } + if stdoutResp != "" { + ch.Write([]byte(stdoutResp)) + } + ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{exitStatus})) + ch.Close() + return + } + req.Reply(false, nil) + } + }() + } + }() + return srv +} + +func hostKeyLine(t *testing.T, s ssh.Signer) string { + t.Helper() + return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(s.PublicKey()))) +} + +func newTestClient(t *testing.T, addr, hostKey string, clientPEM []byte) *Client { + t.Helper() + c, err := New(Config{ + Addr: addr, User: "felhom-peersync", PrivateKey: clientPEM, + HostKeyLine: hostKey, Timeout: 5 * time.Second, + }, log.New(io.Discard, "", 0)) + if err != nil { + t.Fatalf("tenantsync.New: %v", err) + } + return c +} + +const okResp = `{"status":"ok","token_id":"felhom@pbs!peti","token_secret":"s3cr3t-uuid-value",` + + `"fingerprint":"aa:bb","datastore":"felhom-offsite","namespace":"peti"}` + +func TestProvision_RoundTripParsesResult(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0) + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + res, err := c.Provision(context.Background(), "peti") + if err != nil { + t.Fatalf("Provision: %v", err) + } + if res.TokenID != "felhom@pbs!peti" || res.TokenSecret != "s3cr3t-uuid-value" || + res.Fingerprint != "aa:bb" || res.Datastore != "felhom-offsite" || res.Namespace != "peti" { + t.Errorf("Result mis-parsed: %+v", res) + } + srv.mu.Lock() + defer srv.mu.Unlock() + if want := `{"customer_id":"peti","op":"provision"}`; string(srv.captured) != want { + t.Errorf("server received %q, want %q", srv.captured, want) + } + if srv.cmd != "felhom-tenantsync" { + t.Errorf("exec command = %q, want felhom-tenantsync", srv.cmd) + } +} + +func TestProvision_TokenExistsIsTypedError(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + // The script emits the error JSON on stdout AND exits 1 — both must map to ErrTokenExists. + srv := startTestServer(t, hostSigner, clientSigner, + `{"status":"error","code":"token_exists","error":"token felhom@pbs!peti already exists"}`, "", 1) + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + _, err := c.Provision(context.Background(), "peti") + if !errors.Is(err, ErrTokenExists) { + t.Fatalf("err = %v, want ErrTokenExists", err) + } +} + +func TestReissue_SendsReissueOp(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0) + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + if _, err := c.Reissue(context.Background(), "peti"); err != nil { + t.Fatalf("Reissue: %v", err) + } + srv.mu.Lock() + defer srv.mu.Unlock() + if want := `{"customer_id":"peti","op":"reissue"}`; string(srv.captured) != want { + t.Errorf("server received %q, want %q", srv.captured, want) + } +} + +func TestFingerprint_Op(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, `{"status":"ok","fingerprint":"cc:dd"}`, "", 0) + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + fp, err := c.Fingerprint(context.Background()) + if err != nil { + t.Fatalf("Fingerprint: %v", err) + } + if fp != "cc:dd" { + t.Errorf("fingerprint = %q, want cc:dd", fp) + } +} + +// TestErrors_NeverEmbedStdout is the secret-hygiene contract: stdout may carry the token secret, +// so NO error path may quote it. A malformed-but-secret-bearing stdout must yield an error that +// does not contain the marker bytes. (wgsync quotes stdout in its malformed error — this package +// deliberately must not; that divergence is the point of this test.) +func TestErrors_NeverEmbedStdout(t *testing.T) { + cases := []struct { + name string + stdout string + exitStatus uint32 + }{ + {"malformed-ok-exit", `garbage SECRET-MARKER garbage`, 0}, + {"malformed-fail-exit", `partial {"token_secret":"SECRET-MARKER"`, 1}, + {"wrong-status", `{"status":"weird","token_secret":"SECRET-MARKER"}`, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, tc.stdout, "some stderr", tc.exitStatus) + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + _, err := c.Provision(context.Background(), "peti") + if err == nil { + t.Fatal("Provision accepted a malformed response") + } + if strings.Contains(err.Error(), "SECRET-MARKER") { + t.Errorf("error %q embeds stdout bytes — the secret channel leaked into logs", err) + } + }) + } +} + +func TestOkResponseMissingFieldsRefused(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, + `{"status":"ok","token_id":"felhom@pbs!peti"}`, "", 0) // no secret/ns/fp/ds + c := newTestClient(t, srv.addr, hostKeyLine(t, hostSigner), clientPEM) + + if _, err := c.Provision(context.Background(), "peti"); err == nil { + t.Fatal("Provision accepted an ok-response with missing fields") + } +} + +func TestWrongHostKeyRefused(t *testing.T) { + clientPEM, clientSigner := testKeys(t) + _, hostSigner := testKeys(t) + _, otherSigner := testKeys(t) + srv := startTestServer(t, hostSigner, clientSigner, okResp, "", 0) + c := newTestClient(t, srv.addr, hostKeyLine(t, otherSigner), clientPEM) + + _, err := c.Provision(context.Background(), "peti") + if err == nil { + t.Fatal("Provision succeeded against a server with the WRONG host key — the pin is dead") + } + srv.mu.Lock() + defer srv.mu.Unlock() + if len(srv.captured) != 0 { + t.Errorf("payload leaked to a mis-keyed server: %q", srv.captured) + } +} + +func TestInvalidCustomerIDRefusedClientSide(t *testing.T) { + clientPEM, _ := testKeys(t) + _, hostSigner := testKeys(t) + // No server needed — the refusal must happen before any dial. + c := newTestClient(t, "127.0.0.1:1", hostKeyLine(t, hostSigner), clientPEM) + for _, bad := range []string{"", "-leading-dash", ".dot", "has space", strings.Repeat("x", 40)} { + if _, err := c.Provision(context.Background(), bad); err == nil { + t.Errorf("customer_id %q accepted", bad) + } + } +} + +func TestNew_BadInputsFailEarly(t *testing.T) { + clientPEM, _ := testKeys(t) + _, hostSigner := testKeys(t) + hk := hostKeyLine(t, hostSigner) + logger := log.New(io.Discard, "", 0) + + if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: []byte("not-a-key"), HostKeyLine: hk}, logger); err == nil { + t.Error("bad private key accepted") + } + if _, err := New(Config{Addr: "x:22", User: "u", PrivateKey: clientPEM, HostKeyLine: "not a key line"}, logger); err == nil { + t.Error("bad host key line accepted") + } + if _, err := New(Config{User: "u", PrivateKey: clientPEM, HostKeyLine: hk}, logger); err == nil { + t.Error("missing addr accepted") + } +} diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index fca03a6..d82559c 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -413,12 +413,14 @@ func (s *Server) handleConfigNewForm(w http.ResponseWriter, r *http.Request) { ActiveNav string Error string CSRFField template.HTML + PBSDR pbsDRView }{ IsNew: true, Config: &store.CustomerConfig{}, Overrides: make(map[string]interface{}), ActiveNav: "configs", CSRFField: s.csrfField(r), + PBSDR: s.pbsDRViewFor(""), } s.templates.ExecuteTemplate(w, "config_form.html", data) } @@ -484,6 +486,13 @@ func (s *Server) handleConfigCreate(w http.ResponseWriter, r *http.Request) { return } + // PBS DR tier (fail-closed, same discipline; the descriptor lives in the HOST desired-state). + if err := s.applyPBSDR(r.Context(), r, cfg); err != nil { + s.logger.Printf("[ERROR] pbsdr provision for %s: %v", customerID, err) + http.Error(w, "PBS DR provisioning failed: "+err.Error(), http.StatusBadGateway) + return + } + if err := s.store.SaveCustomerConfig(cfg); err != nil { s.logger.Printf("[ERROR] Failed to save config for %s: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) @@ -512,12 +521,14 @@ func (s *Server) handleConfigEditForm(w http.ResponseWriter, r *http.Request, cu ActiveNav string Error string CSRFField template.HTML + PBSDR pbsDRView }{ IsNew: false, Config: cfg, Overrides: overrides, ActiveNav: "configs", CSRFField: s.csrfField(r), + PBSDR: s.pbsDRViewFor(customerID), } s.templates.ExecuteTemplate(w, "config_form.html", data) } @@ -546,6 +557,14 @@ func (s *Server) handleConfigUpdate(w http.ResponseWriter, r *http.Request, cust return } + // PBS DR tier (fail-closed; idempotent on an already-provisioned descriptor — no re-key, + // no second secret, no spurious generation bump). + if err := s.applyPBSDR(r.Context(), r, cfg); err != nil { + s.logger.Printf("[ERROR] pbsdr provision for %s: %v", customerID, err) + http.Error(w, "PBS DR provisioning failed: "+err.Error(), http.StatusBadGateway) + return + } + if err := s.store.SaveCustomerConfig(cfg); err != nil { s.logger.Printf("[ERROR] Failed to update config for %s: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) @@ -897,6 +916,7 @@ func (s *Server) renderConfigForm(w http.ResponseWriter, r *http.Request, isNew ActiveNav string Error string CSRFField template.HTML + PBSDR pbsDRView }{ IsNew: isNew, Config: cfg, @@ -904,6 +924,7 @@ func (s *Server) renderConfigForm(w http.ResponseWriter, r *http.Request, isNew ActiveNav: "configs", Error: errMsg, CSRFField: s.csrfField(r), + PBSDR: s.pbsDRViewFor(cfg.CustomerID), } s.templates.ExecuteTemplate(w, "config_form.html", data) } diff --git a/hub/internal/web/configs_debug_test.go b/hub/internal/web/configs_debug_test.go index 0b55481..a597e89 100644 --- a/hub/internal/web/configs_debug_test.go +++ b/hub/internal/web/configs_debug_test.go @@ -154,6 +154,7 @@ func TestConfigForm_DebugRenderState(t *testing.T) { ActiveNav string Error string CSRFField string + PBSDR pbsDRView }{ Config: &store.CustomerConfig{CustomerID: "c1"}, Overrides: overrides, diff --git a/hub/internal/web/pbsdr.go b/hub/internal/web/pbsdr.go new file mode 100644 index 0000000..4a8a3c7 --- /dev/null +++ b/hub/internal/web/pbsdr.go @@ -0,0 +1,308 @@ +package web + +// PBS DR tier (SLICE 1): the hub-side provisioning flow behind the customer config form's +// "PBS DR tier (ep0)" section. On enable+save: verify the host's WG peer exists (the agent +// self-registers its pubkey — absence is fail-closed, nothing to allocate here) → provision the +// per-customer ep0 tenancy over the tenantsync channel → store the token secret CONSUME-ONCE +// (host-scoped; the agent fetches it via POST /api/v1/hosts/{id}/pbs/consume-token) → merge the +// NON-SECRET descriptor into the host's desired_json under "pbs_dr" (the admin-set path; +// SetHostDesired bumps the generation, which is the agent's change signal). +// +// Fail-closed like applyOffsite: any error means NO descriptor write, NO bump, NO half-enabled +// state. Idempotent: an already-provisioned descriptor short-circuits — no tenantsync call, no +// second secret, no spurious bump. Disable rewrites the descriptor with enabled=false (bump) — +// the ep0 namespace/token are NOT deprovisioned (the offsite-disable precedent: data deletion is +// a deliberate, separate decision). + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" +) + +// tenancyProvisioner is the tenantsync seam — satisfied by *tenantsync.Client; tests inject a fake. +type tenancyProvisioner interface { + Provision(ctx context.Context, customerID string) (*tenantsync.Result, error) + Reissue(ctx context.Context, customerID string) (*tenantsync.Result, error) +} + +// SetTenantSync enables PBS DR tier provisioning (optional). Without it, saving a config with the +// tier enabled returns an error (not configured on this hub); the form section still renders. +func (s *Server) SetTenantSync(p tenancyProvisioner) { s.tenantsync = p } + +// pbsDRDescriptor is the NON-SECRET pbs_dr block in a host's desired_json. It NEVER carries the +// token secret (that is host_pbs_secrets custody, consume-once). +type pbsDRDescriptor struct { + Enabled bool `json:"enabled"` + StorageID string `json:"storage_id,omitempty"` + PBSTunnelIP string `json:"pbs_tunnel_ip,omitempty"` + Datastore string `json:"datastore,omitempty"` + Namespace string `json:"namespace,omitempty"` + TokenID string `json:"token_id,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` +} + +// defaultPBSStorageID is the storage-entry id the agent bridge creates on a customer box. The DEMO +// host's adopted manual entry is `felhom-offsite` — the descriptor carries the id so the bridge is +// name-agnostic and the felhom-pbs collision on the demo dissolves (spike naming note). +const defaultPBSStorageID = "felhom-pbs" + +// pveStorageIDRe validates the operator-typed storage id (PVE storage-id grammar, conservative). +var pveStorageIDRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.-]{0,27}$`) + +// readPBSDR extracts the pbs_dr descriptor from a host's desired_json ("" / absent → nil). +func readPBSDR(desiredJSON string) *pbsDRDescriptor { + var doc struct { + PBSDR *pbsDRDescriptor `json:"pbs_dr"` + } + if err := json.Unmarshal([]byte(desiredJSON), &doc); err != nil { + return nil + } + return doc.PBSDR +} + +// mergePBSDR merges the descriptor under the "pbs_dr" key of a desired_json object, preserving +// every other key (the operator blob, dr blocks…). Same shape as offsite.MergeDescriptor. +func mergePBSDR(desiredJSON string, d *pbsDRDescriptor) (string, error) { + obj := map[string]json.RawMessage{} + if strings.TrimSpace(desiredJSON) != "" && desiredJSON != "{}" { + if err := json.Unmarshal([]byte(desiredJSON), &obj); err != nil { + return "", fmt.Errorf("pbsdr: parse desired_json: %w", err) + } + } + db, err := json.Marshal(d) + if err != nil { + return "", err + } + obj["pbs_dr"] = db + out, err := json.Marshal(obj) + if err != nil { + return "", err + } + return string(out), nil +} + +// applyPBSDR handles the config form's PBS DR section on create/update. Called BEFORE +// SaveCustomerConfig (fail-closed: an error must abort the whole save). The descriptor lives in +// the HOST's desired_json — ConfigJSON never carries pbs_dr (single source of truth). +func (s *Server) applyPBSDR(ctx context.Context, r *http.Request, cfg *store.CustomerConfig) error { + enabled := r.FormValue("pbsdr_enabled") == "on" || r.FormValue("pbsdr_enabled") == "true" + storageID := strings.TrimSpace(r.FormValue("pbsdr_storage_id")) + if storageID == "" { + storageID = defaultPBSStorageID + } + + host, err := s.store.GetHostByCustomer(cfg.CustomerID) + if err != nil { + return fmt.Errorf("pbsdr: host lookup: %w", err) + } + if host == nil { + if !enabled { + return nil // nothing enrolled, nothing enabled — nothing to do + } + return fmt.Errorf("no host enrolled for customer %s yet — the PBS DR tier needs the enrolled host", cfg.CustomerID) + } + cur := readPBSDR(host.DesiredJSON) + + if !enabled { + // Disable = descriptor enabled:false (coords kept; NO ep0 deprovision). Only when there is + // something to disable — otherwise a pure no-op (no spurious bump). + if cur == nil || !cur.Enabled { + return nil + } + cur.Enabled = false + merged, err := mergePBSDR(host.DesiredJSON, cur) + if err != nil { + return err + } + gen, err := s.store.SetHostDesired(host.HostID, []byte(merged)) + if err != nil { + return fmt.Errorf("pbsdr: desired-state write: %w", err) + } + s.logger.Printf("[INFO] pbsdr disabled for %s (host %s, gen %d; ep0 tenancy kept)", cfg.CustomerID, host.HostID, gen) + return nil + } + + if !pveStorageIDRe.MatchString(storageID) { + return fmt.Errorf("invalid PBS storage id %q", storageID) + } + + // Already provisioned → success-no-op (descriptor re-served unchanged; NO new secret, no + // tenantsync call). Only an actual change (re-enable, storage id edit) rewrites + bumps. + if cur != nil && cur.Namespace != "" { + if cur.Enabled && cur.StorageID == storageID { + return nil + } + cur.Enabled = true + cur.StorageID = storageID + merged, err := mergePBSDR(host.DesiredJSON, cur) + if err != nil { + return err + } + gen, err := s.store.SetHostDesired(host.HostID, []byte(merged)) + if err != nil { + return fmt.Errorf("pbsdr: desired-state write: %w", err) + } + s.logger.Printf("[INFO] pbsdr descriptor updated for %s (host %s, gen %d; tenancy unchanged)", cfg.CustomerID, host.HostID, gen) + return nil + } + + // Fresh provision. Fail-closed preconditions first. + if s.tenantsync == nil { + return fmt.Errorf("PBS DR provisioning is not configured on this hub (no tenantsync key)") + } + if _, err := s.store.GetWGPeerForHost(host.HostID); err == sql.ErrNoRows { + return fmt.Errorf("host %s has not reported a WG key yet — the tunnel peer must exist before the PBS DR tier", host.HostID) + } else if err != nil { + return fmt.Errorf("pbsdr: wg peer lookup: %w", err) + } + ep, err := s.store.GetWGEndpoint() + if err == sql.ErrNoRows { + return fmt.Errorf("wg endpoint not configured — register ep0 before the PBS DR tier") + } else if err != nil { + return fmt.Errorf("pbsdr: wg endpoint read: %w", err) + } + + // Detach from the request context (the applyOffsite F1 precedent): once provisioning starts, + // provision→store-secret→descriptor must complete; an impatient re-click must not cancel + // between the ep0 mutation and the consume-once store write (a stranded token would need a + // manual re-issue). The absolute timeout still bounds a hung SSH exec. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + res, err := s.tenantsync.Provision(ctx, cfg.CustomerID) + if errors.Is(err, tenantsync.ErrTokenExists) { + // ep0 has a token but the hub has no descriptor — state mismatch (lost hub state or a + // half-torn earlier attempt). Never silently re-key: the operator decides via Re-issue. + return fmt.Errorf("the endpoint already holds a PBS token for %s but the hub has no descriptor — use the explicit \"Re-issue PBS credentials\" action", cfg.CustomerID) + } + if err != nil { + return err + } + + // The atom: secret first (consume-once custody), then descriptor+bump (the agent's signal). + if err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret); err != nil { + return fmt.Errorf("pbsdr: store one-time token secret: %w", err) + } + desc := &pbsDRDescriptor{ + Enabled: true, + StorageID: storageID, + PBSTunnelIP: ep.PBSTunnelIP, + Datastore: res.Datastore, + Namespace: res.Namespace, + TokenID: res.TokenID, + Fingerprint: res.Fingerprint, + } + merged, err := mergePBSDR(host.DesiredJSON, desc) + if err != nil { + return err + } + gen, err := s.store.SetHostDesired(host.HostID, []byte(merged)) + if err != nil { + return fmt.Errorf("pbsdr: desired-state write: %w", err) + } + s.logger.Printf("[INFO] pbsdr provisioned for %s (host %s, ns %s, token_id %s, gen %d; secret stored consume-once, withheld from logs)", + cfg.CustomerID, host.HostID, res.Namespace, res.TokenID, gen) + return nil +} + +// handlePBSDRReissue explicitly re-keys the customer's ep0 PBS token (the offsite F4 precedent): +// tenantsync reissue → fresh consume-once secret → descriptor refresh + generation bump so the +// agent re-runs its bridge and consumes the fresh secret. The secret value is never logged. +func (s *Server) handlePBSDRReissue(w http.ResponseWriter, r *http.Request, customerID string) { + if s.tenantsync == nil { + http.Error(w, "PBS DR provisioning is not configured on this hub", http.StatusBadGateway) + return + } + cfg, err := s.store.GetCustomerConfig(customerID) + if err != nil || cfg == nil { + http.NotFound(w, r) + return + } + host, err := s.store.GetHostByCustomer(customerID) + if err != nil || host == nil { + http.Error(w, "No host enrolled for this customer", http.StatusBadRequest) + return + } + cur := readPBSDR(host.DesiredJSON) + if cur == nil || cur.Namespace == "" { + http.Error(w, "No provisioned PBS DR tier for this customer", http.StatusBadRequest) + return + } + + // Same detached-ctx discipline as applyPBSDR: reissue→store→bump is the atom. + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 2*time.Minute) + defer cancel() + res, err := s.tenantsync.Reissue(ctx, customerID) + if err != nil { + s.logger.Printf("[ERROR] pbsdr reissue for %s: %v", customerID, err) + http.Error(w, "PBS credential re-issue failed: "+err.Error(), http.StatusBadGateway) + return + } + if err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret); err != nil { + s.logger.Printf("[ERROR] pbsdr reissue for %s: secret store: %v", customerID, err) + http.Error(w, "Re-issued on the endpoint but storing the secret failed — re-issue again", http.StatusInternalServerError) + return + } + cur.TokenID = res.TokenID + cur.Fingerprint = res.Fingerprint + cur.Datastore = res.Datastore + cur.Namespace = res.Namespace + merged, err := mergePBSDR(host.DesiredJSON, cur) + if err == nil { + _, err = s.store.SetHostDesired(host.HostID, []byte(merged)) + } + if err != nil { + s.logger.Printf("[ERROR] pbsdr reissue for %s: descriptor bump: %v", customerID, err) + http.Error(w, "Credential re-issued but the descriptor bump failed — save the config once to trigger the pickup", http.StatusInternalServerError) + return + } + s.logger.Printf("[INFO] pbsdr credentials re-issued for %s (host %s; fresh consume-once secret stored)", customerID, host.HostID) + http.Redirect(w, r, "/customers/"+customerID+"?flash=pbsdr_reissued", http.StatusSeeOther) +} + +// pbsDRView is the config form's render model for the PBS DR section. +type pbsDRView struct { + Supported bool // tenantsync configured on this hub + NoHost bool // no enrolled host for the customer (enable would fail-closed) + HostID string + Enabled bool + StorageID string // current or the default + Provisioned bool + Namespace string + TokenID string +} + +// pbsDRViewFor loads the section state for the form. Read-only; every error degrades to a +// zero-ish view (the section still renders). +func (s *Server) pbsDRViewFor(customerID string) pbsDRView { + v := pbsDRView{Supported: s.tenantsync != nil, StorageID: defaultPBSStorageID} + if customerID == "" { + v.NoHost = true + return v + } + host, err := s.store.GetHostByCustomer(customerID) + if err != nil || host == nil { + v.NoHost = true + return v + } + v.HostID = host.HostID + if d := readPBSDR(host.DesiredJSON); d != nil { + v.Enabled = d.Enabled + if d.StorageID != "" { + v.StorageID = d.StorageID + } + v.Provisioned = d.Namespace != "" + v.Namespace = d.Namespace + v.TokenID = d.TokenID + } + return v +} diff --git a/hub/internal/web/pbsdr_test.go b/hub/internal/web/pbsdr_test.go new file mode 100644 index 0000000..2b4b0ba --- /dev/null +++ b/hub/internal/web/pbsdr_test.go @@ -0,0 +1,373 @@ +package web + +// PBS DR SLICE 1 — the provisioning flow behind the config form, against a FAKE tenancy +// provisioner (no SSH in CI). The load-bearing contracts: descriptor lands in the HOST +// desired_json + generation bump; the secret is stored consume-once and appears NOWHERE else +// (ConfigJSON, desired-state, logs); every failure is fail-closed (no descriptor, no bump, no +// secret); an already-provisioned re-save is a pure no-op (no re-key, no second secret, no +// spurious bump); re-issue rotates the secret + bumps. + +import ( + "bytes" + "context" + "database/sql" + "errors" + "log" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" +) + +type fakeTenancy struct { + provisionCalls int + reissueCalls int + err error + secret string +} + +func (f *fakeTenancy) result(customerID string) *tenantsync.Result { + return &tenantsync.Result{ + TokenID: "felhom@pbs!" + customerID, + TokenSecret: f.secret, + Fingerprint: "aa:bb:cc", + Datastore: "felhom-offsite", + Namespace: customerID, + } +} + +func (f *fakeTenancy) Provision(ctx context.Context, customerID string) (*tenantsync.Result, error) { + f.provisionCalls++ + if f.err != nil { + return nil, f.err + } + return f.result(customerID), nil +} + +func (f *fakeTenancy) Reissue(ctx context.Context, customerID string) (*tenantsync.Result, error) { + f.reissueCalls++ + if f.err != nil { + return nil, f.err + } + return f.result(customerID), nil +} + +// newPBSDRServer builds a server + store with the full provisioning preconditions satisfied: +// customer config, enrolled host, WG endpoint record, bound WG peer. The logger is captured so +// tests can grep-assert the secret never reaches it. +func newPBSDRServer(t *testing.T, fake *fakeTenancy) (*Server, *store.Store, *bytes.Buffer) { + t.Helper() + s, st := newTestServer(t) + logBuf := &bytes.Buffer{} + s.logger = log.New(logBuf, "", 0) + if fake != nil { + s.SetTenantSync(fake) + } + if err := st.SaveCustomerConfig(&store.CustomerConfig{ + CustomerID: "peti", APIKey: "capi", RetrievalPassword: "pw", + }); err != nil { + t.Fatalf("seed config: %v", err) + } + if err := st.UpsertHost(&store.Host{HostID: "peti-01", CustomerID: "peti", APIKey: "hapi"}); err != nil { + t.Fatalf("seed host: %v", err) + } + if err := st.SetWGEndpoint(&store.WGEndpoint{ + EndpointID: "ep0", DNSName: "ep0.felhom.eu", WGPort: 443, + ServerPubkey: "SPK", TunnelSubnet: "10.77.0.0/24", PBSTunnelIP: "10.77.0.1", + }); err != nil { + t.Fatalf("seed endpoint: %v", err) + } + if _, _, err := st.RegisterWGPeerForHost("peti-01", "PETIPUBKEY"); err != nil { + t.Fatalf("seed wg peer: %v", err) + } + return s, st, logBuf +} + +// postUpdate drives the REAL handler pipeline (handleConfigUpdate → applyPBSDR → save). +func postUpdate(t *testing.T, s *Server, form url.Values) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest("POST", "/configs/peti", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rr := httptest.NewRecorder() + s.handleConfigUpdate(rr, req, "peti") + return rr +} + +func hostState(t *testing.T, st *store.Store) (desc *pbsDRDescriptor, desiredJSON string, gen int64) { + t.Helper() + h, err := st.GetHost("peti-01") + if err != nil || h == nil { + t.Fatalf("host read: %v", err) + } + return readPBSDR(h.DesiredJSON), h.DesiredJSON, h.DesiredGeneration +} + +func TestPBSDR_ProvisionHappyPath(t *testing.T) { + fake := &fakeTenancy{secret: "SUPER-SECRET-TOKEN"} + s, st, logBuf := newPBSDRServer(t, fake) + + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}, "pbsdr_storage_id": {"felhom-pbs"}}) + if rr.Code != 303 { + t.Fatalf("save = %d (%s), want 303", rr.Code, rr.Body.String()) + } + if fake.provisionCalls != 1 { + t.Errorf("provision calls = %d, want 1", fake.provisionCalls) + } + + desc, desiredJSON, gen := hostState(t, st) + if desc == nil { + t.Fatalf("no pbs_dr descriptor in desired_json: %s", desiredJSON) + } + if !desc.Enabled || desc.StorageID != "felhom-pbs" || desc.PBSTunnelIP != "10.77.0.1" || + desc.Datastore != "felhom-offsite" || desc.Namespace != "peti" || + desc.TokenID != "felhom@pbs!peti" || desc.Fingerprint != "aa:bb:cc" { + t.Errorf("descriptor wrong: %+v", desc) + } + if gen != 1 { + t.Errorf("desired_generation = %d, want 1 (exactly one bump)", gen) + } + + // The secret: stored consume-once for the HOST… + got, err := st.ConsumeHostPBSSecret("peti-01") + if err != nil || got != "SUPER-SECRET-TOKEN" { + t.Fatalf("consume = (%q, %v), want the stored secret", got, err) + } + // …and NOWHERE else: not in the desired-state, not in ConfigJSON, not in any log line. + if strings.Contains(desiredJSON, "SUPER-SECRET-TOKEN") { + t.Error("secret leaked into desired_json") + } + cfg, _ := st.GetCustomerConfig("peti") + if strings.Contains(cfg.ConfigJSON, "SUPER-SECRET-TOKEN") || strings.Contains(cfg.ConfigJSON, "pbs_dr") { + t.Errorf("ConfigJSON must carry neither the secret nor the descriptor: %s", cfg.ConfigJSON) + } + if strings.Contains(logBuf.String(), "SUPER-SECRET-TOKEN") { + t.Error("secret leaked into the hub log") + } +} + +func TestPBSDR_ResaveIsNoOp(t *testing.T) { + fake := &fakeTenancy{secret: "SUPER-SECRET-TOKEN"} + s, st, _ := newPBSDRServer(t, fake) + + postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if _, err := st.ConsumeHostPBSSecret("peti-01"); err != nil { + t.Fatalf("first secret consume: %v", err) + } + + // Idempotent re-save: NO re-key, NO second secret row, NO spurious generation bump. + // (Red-proof: dropping the already-provisioned short-circuit in applyPBSDR re-runs + // Provision → calls=2 + a fresh consumable secret → FAIL.) + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if rr.Code != 303 { + t.Fatalf("re-save = %d, want 303 (already-provisioned is success-no-op)", rr.Code) + } + if fake.provisionCalls != 1 { + t.Errorf("provision calls after re-save = %d, want 1 (token must not rotate)", fake.provisionCalls) + } + if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows { + t.Errorf("re-save created a fresh secret (consume err %v, want ErrNoRows)", err) + } + if _, _, gen := hostState(t, st); gen != 1 { + t.Errorf("generation after re-save = %d, want 1 (no spurious bump)", gen) + } +} + +func TestPBSDR_FailClosed(t *testing.T) { + t.Run("no WG peer", func(t *testing.T) { + fake := &fakeTenancy{secret: "S"} + s, st, _ := newPBSDRServer(t, fake) + // Kill the precondition: the host has never registered a WG key. + if err := st.RemoveWGPeer("PETIPUBKEY"); err != nil { + t.Fatalf("remove seed peer: %v", err) + } + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if rr.Code != 502 { + t.Fatalf("save without WG peer = %d, want 502", rr.Code) + } + if fake.provisionCalls != 0 { + t.Errorf("provision reached despite missing peer (%d calls)", fake.provisionCalls) + } + h, _ := st.GetHost("peti-01") + if readPBSDR(h.DesiredJSON) != nil || h.DesiredGeneration != 0 { + t.Errorf("fail-closed violated: desc=%v gen=%d", readPBSDR(h.DesiredJSON), h.DesiredGeneration) + } + }) + + t.Run("tenantsync error", func(t *testing.T) { + fake := &fakeTenancy{err: errors.New("ssh boom")} + s, st, _ := newPBSDRServer(t, fake) + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if rr.Code != 502 { + t.Fatalf("save with tenantsync error = %d, want 502", rr.Code) + } + desc, _, gen := hostState(t, st) + if desc != nil || gen != 0 { + t.Errorf("fail-closed violated: desc=%+v gen=%d", desc, gen) + } + if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows { + t.Errorf("a secret exists after a failed provision (err %v)", err) + } + // Fail-closed also means the config save itself was aborted. + cfg, _ := st.GetCustomerConfig("peti") + if cfg.CustomerName == "half-saved" { + t.Error("config row saved despite provisioning failure") + } + }) + + t.Run("token_exists points at re-issue", func(t *testing.T) { + fake := &fakeTenancy{err: tenantsync.ErrTokenExists} + s, st, _ := newPBSDRServer(t, fake) + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if rr.Code != 502 || !strings.Contains(rr.Body.String(), "Re-issue") { + t.Fatalf("token_exists = %d (%s), want 502 mentioning Re-issue", rr.Code, rr.Body.String()) + } + if desc, _, gen := hostState(t, st); desc != nil || gen != 0 { + t.Errorf("state written on token_exists: desc=%+v gen=%d", desc, gen) + } + _ = st + }) + + t.Run("not configured", func(t *testing.T) { + s, st, _ := newPBSDRServer(t, nil) // no tenantsync + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + if rr.Code != 502 { + t.Fatalf("save without tenantsync = %d, want 502", rr.Code) + } + if desc, _, gen := hostState(t, st); desc != nil || gen != 0 { + t.Errorf("state written without a provisioner: desc=%+v gen=%d", desc, gen) + } + }) +} + +func TestPBSDR_DisableKeepsTenancy(t *testing.T) { + fake := &fakeTenancy{secret: "S"} + s, st, _ := newPBSDRServer(t, fake) + postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + + // Unchecked box → descriptor enabled:false, coords kept, ONE bump; no endpoint mutation. + rr := postUpdate(t, s, url.Values{}) + if rr.Code != 303 { + t.Fatalf("disable save = %d, want 303", rr.Code) + } + desc, _, gen := hostState(t, st) + if desc == nil || desc.Enabled || desc.Namespace != "peti" || desc.TokenID == "" { + t.Fatalf("disable must keep coords with enabled=false: %+v", desc) + } + if gen != 2 { + t.Errorf("generation = %d, want 2 (enable + disable)", gen) + } + // A second disabled save is a pure no-op. + postUpdate(t, s, url.Values{}) + if _, _, gen := hostState(t, st); gen != 2 { + t.Errorf("second disabled save bumped generation to %d", gen) + } + if fake.provisionCalls != 1 || fake.reissueCalls != 0 { + t.Errorf("endpoint touched on disable: provision=%d reissue=%d", fake.provisionCalls, fake.reissueCalls) + } +} + +func TestPBSDR_StorageIDChangeUpdatesDescriptorOnly(t *testing.T) { + fake := &fakeTenancy{secret: "S"} + s, st, _ := newPBSDRServer(t, fake) + postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + st.ConsumeHostPBSSecret("peti-01") // spend the provision secret + + rr := postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}, "pbsdr_storage_id": {"felhom-offsite"}}) + if rr.Code != 303 { + t.Fatalf("storage-id change = %d, want 303", rr.Code) + } + desc, _, gen := hostState(t, st) + if desc.StorageID != "felhom-offsite" || !desc.Enabled { + t.Errorf("descriptor not updated: %+v", desc) + } + if gen != 2 { + t.Errorf("generation = %d, want 2", gen) + } + if fake.provisionCalls != 1 { + t.Errorf("a descriptor edit re-provisioned the tenancy (%d calls)", fake.provisionCalls) + } + if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows { + t.Errorf("a descriptor edit staged a new secret (err %v)", err) + } +} + +func TestPBSDR_Reissue(t *testing.T) { + fake := &fakeTenancy{secret: "OLD-SECRET"} + s, st, logBuf := newPBSDRServer(t, fake) + postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + st.ConsumeHostPBSSecret("peti-01") // agent already consumed; the dead-end scenario + + fake.secret = "FRESH-SECRET" + req := httptest.NewRequest("POST", "/configs/peti/pbsdr-reissue", nil) + rr := httptest.NewRecorder() + s.handlePBSDRReissue(rr, req, "peti") + if rr.Code != 303 { + t.Fatalf("reissue = %d (%s), want 303", rr.Code, rr.Body.String()) + } + if fake.reissueCalls != 1 { + t.Errorf("reissue calls = %d, want 1", fake.reissueCalls) + } + got, err := st.ConsumeHostPBSSecret("peti-01") + if err != nil || got != "FRESH-SECRET" { + t.Fatalf("fresh secret consume = (%q, %v)", got, err) + } + if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows { + t.Error("the old secret path survived the re-issue") + } + if _, _, gen := hostState(t, st); gen != 2 { + t.Errorf("generation = %d, want 2 (provision + reissue)", gen) + } + if strings.Contains(logBuf.String(), "FRESH-SECRET") || strings.Contains(logBuf.String(), "OLD-SECRET") { + t.Error("secret leaked into the hub log") + } +} + +func TestPBSDR_ReissueRequiresProvisionedState(t *testing.T) { + fake := &fakeTenancy{secret: "S"} + s, _, _ := newPBSDRServer(t, fake) + req := httptest.NewRequest("POST", "/configs/peti/pbsdr-reissue", nil) + rr := httptest.NewRecorder() + s.handlePBSDRReissue(rr, req, "peti") + if rr.Code != 400 { + t.Fatalf("reissue without a provisioned tier = %d, want 400", rr.Code) + } + if fake.reissueCalls != 0 { + t.Errorf("endpoint touched without a descriptor (%d calls)", fake.reissueCalls) + } +} + +// The form render leg: the section reflects the provisioned descriptor (checkbox + namespace line) +// and the fresh-form default storage id. +func TestPBSDR_FormRendersState(t *testing.T) { + fake := &fakeTenancy{secret: "S"} + s, _, _ := newPBSDRServer(t, fake) + postUpdate(t, s, url.Values{"pbsdr_enabled": {"on"}}) + + req := httptest.NewRequest("GET", "/configs/peti/edit", nil) + rr := httptest.NewRecorder() + s.handleConfigEditForm(rr, req, "peti") + out := rr.Body.String() + if !strings.Contains(out, `name="pbsdr_enabled" checked`) { + t.Error("enabled checkbox not checked after provisioning") + } + if !strings.Contains(out, "Provisioned: namespace peti, token felhom@pbs!peti") { + t.Error("provisioned-state line missing") + } + if !strings.Contains(out, "pbsdr-reissue") { + t.Error("re-issue button missing") + } + if strings.Contains(out, "S\"") && strings.Contains(out, "token_secret") { + t.Error("secret-ish content rendered") + } + + // Deadline check on the detached-ctx behavior is out of scope here; the view path must + // stay read-only — a render must not have touched provisioning. + if fake.provisionCalls != 1 { + t.Errorf("rendering the form called provision (%d calls)", fake.provisionCalls) + } + _ = time.Second +} diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index 11d2f98..bf6de9a 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -59,6 +59,7 @@ type Server struct { assetsMgr *assets.Manager gitea *gitea.Client // optional; enables the Day-0 artifact version dropdowns offsite *offsite.Provisioner // optional; enables Hetzner offsite provisioning (SLICE 1) + tenantsync tenancyProvisioner // optional; enables PBS DR tier provisioning (web/pbsdr.go) sessions map[string]*hubSession sessionsMu sync.RWMutex @@ -367,6 +368,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } else { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) } + case strings.HasPrefix(path, "/configs/") && strings.HasSuffix(path, "/pbsdr-reissue"): + customerID := strings.TrimPrefix(path, "/configs/") + customerID = strings.TrimSuffix(customerID, "/pbsdr-reissue") + if r.Method == http.MethodPost { + s.handlePBSDRReissue(w, r, customerID) + } else { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } case strings.HasPrefix(path, "/configs/") && strings.HasSuffix(path, "/offsite-freeze"): customerID := strings.TrimPrefix(path, "/configs/") customerID = strings.TrimSuffix(customerID, "/offsite-freeze") diff --git a/hub/internal/web/templates/config_form.html b/hub/internal/web/templates/config_form.html index 7ce9338..cd2a994 100644 --- a/hub/internal/web/templates/config_form.html +++ b/hub/internal/web/templates/config_form.html @@ -169,6 +169,36 @@ {{end}}{{end}}{{end}} +
+

PBS DR tier (ep0)

 +
+
+ + {{if not .PBSDR.Supported}} + Not configured on this hub (no tenantsync key) — enabling will fail until it is. + {{else if .PBSDR.NoHost}} + No host enrolled for this customer yet — enabling fails until the agent enrolls and reports its WG key. + {{end}} +
+
+ + + The storage-entry id the agent creates on the box. Default felhom-pbs; the demo host's adopted manual entry is felhom-offsite. +
+
+ {{if .PBSDR.Provisioned}} +

Provisioned: namespace {{.PBSDR.Namespace}}, token {{.PBSDR.TokenID}} (host {{.PBSDR.HostID}}) — the token secret is delivered to the host agent once (never shown here).

+ + + {{end}} +
+
Cancel diff --git a/manifests/hub.yaml b/manifests/hub.yaml index d3cc8d8..4ed198b 100644 --- a/manifests/hub.yaml +++ b/manifests/hub.yaml @@ -171,6 +171,11 @@ spec: name: wg-endpoint-ssh key: hostkey optional: true + # PBS DR tier (SLICE 1): the tenantsync channel — same endpoint + pinned host key as + # peersync (env above), its OWN private key from Secret/tenantsync (out-of-band, + # runbook offsite-endpoint.md §10). Optional: absent → the hub logs tenantsync disabled. + - name: TENANTSYNC_SSH_KEY_FILE + value: "/etc/hub-secrets/tenantsync/key" # Offsite provisioning (SLICE 1+2): Hetzner Storage Box API token + the NUMERIC id of the # pool box, from the out-of-band Secret/storagebox (NOT committed). The token MUST be scoped # to the dedicated storage project — NEVER the shared-project token (it can touch ep0). @@ -206,6 +211,9 @@ spec: - name: wg-endpoint-ssh mountPath: /etc/hub-secrets/wg-endpoint-ssh readOnly: true + - name: tenantsync + mountPath: /etc/hub-secrets/tenantsync + readOnly: true livenessProbe: httpGet: path: /healthz @@ -235,6 +243,14 @@ spec: - key: key path: key mode: 0400 + - name: tenantsync + secret: + secretName: tenantsync + optional: true + items: + - key: key + path: key + mode: 0400 # ============================================================================= # SERVICE diff --git a/scripts/felhom-tenantsync.sh b/scripts/felhom-tenantsync.sh new file mode 100644 index 0000000..ac941e0 --- /dev/null +++ b/scripts/felhom-tenantsync.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# felhom-tenantsync v1.0.0 — the offsite endpoint's per-customer PBS tenancy surface (PBS DR tier +# SLICE 1; spike SPIKE-pbs-tier-provisioning-2026-07-10 §3). +# +# Runs as the SSH forced command for the hub's SECOND `felhom-peersync` key (via sudo — its own +# single sudoers line; the peersync script/key are untouched: one script, one job). JSON on stdin, +# JSON on stdout. Ops: +# +# {"op":"provision","customer_id":""} → ensure namespace (idempotent) → CREATE token +# felhom@pbs! (an EXISTING token is a hard error, code "token_exists" — re-issue is the +# explicit path) → dual-grant DatastoreBackup on /datastore/felhom-offsite/ to BOTH the +# user and the token (PBS privsep = intersection) → self-check: list the namespace AS the +# new token (one regen retry per the spike's transient-403 note; still failing → rollback) +# → {"status":"ok","token_id","token_secret","fingerprint","datastore","namespace"} +# {"op":"reissue","customer_id":""} → delete-token (its ACLs purge with it — spike) → +# recreate → re-grant BOTH → self-check → same ok-shape with the FRESH secret. +# {"op":"fingerprint"} → {"status":"ok","fingerprint":""} +# +# NO deprovision op in slice 1 — namespace/data deletion is a deliberate, separate decision (the +# offsite-disable precedent: disable never destroys data). +# +# Secret hygiene (load-bearing): +# - The token secret exists ONLY in memory and in the final stdout JSON — never a file, never +# stderr (the hub embeds remote stderr in error logs), never an argument. +# - The transient root@pam admin token (namespace ops are client-side) is held in memory and +# deleted on EVERY exit path (trap). A leftover from a crashed run is deleted at entry. +# - All tool stdout is redirected to stderr — the response JSON is the ONLY stdout bytes. +# +# Ordering facts this script encodes (all live-proven in the spike — do not "simplify"): +# - the token must exist BEFORE its ACL grant (PBS validates the auth-id); +# - user delete-token PURGES the token's ACLs → re-issue must re-grant; +# - `user generate-token` has no --output-format → the "value" line is sed-parsed; +# - proxmox-backup-client needs PBS_FINGERPRINT even against localhost. +set -euo pipefail + +DS=felhom-offsite +PBS_USER=felhom@pbs +ADMIN_TOKEN_NAME=tenantsync-admin +REPO_HOST=localhost + +err_json() { # code, message → error JSON on stdout, exit 1 + printf '{"status":"error","code":"%s","error":"%s"}\n' "$1" "$2" + exit 1 +} + +command -v jq >/dev/null || err_json internal "jq is required" +command -v proxmox-backup-manager >/dev/null || err_json internal "proxmox-backup-manager is required" +command -v proxmox-backup-client >/dev/null || err_json internal "proxmox-backup-client is required" + +# 1. Read stdin capped at 64 KiB; validate the envelope BEFORE touching anything. +payload=$(head -c 65536) +[ -n "$payload" ] || err_json bad_request "empty payload" +jq -e 'type == "object" and (.op | type) == "string"' >/dev/null 2>&1 <<<"$payload" \ + || err_json bad_request "payload must be a JSON object with a string op" +OP=$(jq -r '.op' <<<"$payload") + +export PBS_FINGERPRINT +PBS_FINGERPRINT=$(proxmox-backup-manager cert info | awk '/Fingerprint/{print $3}' | head -1) +[ -n "$PBS_FINGERPRINT" ] || err_json internal "could not read the PBS cert fingerprint" + +if [ "$OP" = "fingerprint" ]; then + printf '{"status":"ok","fingerprint":"%s"}\n' "$PBS_FINGERPRINT" + exit 0 +fi + +case "$OP" in provision|reissue) ;; *) err_json bad_request "unknown op" ;; esac + +# customer_id → the namespace AND the token name. Conservative charset (PBS ns + token grammar, +# no leading dash/dot so it can never parse as an option). +CID=$(jq -r '.customer_id // ""' <<<"$payload") +[[ "$CID" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}$ ]] \ + || err_json bad_request "customer_id must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,30}\$" +TOKEN_ID="$PBS_USER!$CID" + +log() { echo "felhom-tenantsync: $*" >&2; } + +# 2. Transient admin token for the client-side namespace ops. Deleted on every exit; a leftover +# from a crashed run is cleared first (generate-token fails on an existing name). +proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true +ADM=$(proxmox-backup-manager user generate-token root@pam "$ADMIN_TOKEN_NAME" \ + | sed -n 's/.*"value": "\([^"]*\)".*/\1/p') +[ -n "$ADM" ] || err_json internal "admin token generation failed" +cleanup_admin() { + proxmox-backup-manager user delete-token root@pam "$ADMIN_TOKEN_NAME" >&2 2>/dev/null || true + proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \ + --auth-id "root@pam!$ADMIN_TOKEN_NAME" --delete >&2 2>/dev/null || true +} +trap cleanup_admin EXIT +proxmox-backup-manager acl update "/datastore/$DS" DatastoreAdmin \ + --auth-id "root@pam!$ADMIN_TOKEN_NAME" >&2 + +ADMIN_REPO="root@pam!$ADMIN_TOKEN_NAME@$REPO_HOST:$DS" + +# 3. Ensure the shared user + the namespace (both idempotent). +if ! proxmox-backup-manager user list --output-format json | jq -e --arg u "$PBS_USER" \ + 'any(.[]; .userid == $u)' >/dev/null; then + proxmox-backup-manager user create "$PBS_USER" --comment 'offsite tenancy' >&2 + log "created user $PBS_USER" +fi +if ! PBS_PASSWORD="$ADM" proxmox-backup-client namespace list --repository "$ADMIN_REPO" \ + --output-format json | jq -e --arg ns "$CID" 'any(.[]; .ns == $ns)' >/dev/null; then + PBS_PASSWORD="$ADM" proxmox-backup-client namespace create "$CID" --repository "$ADMIN_REPO" >&2 + log "created namespace $CID" +fi + +token_exists() { + proxmox-backup-manager user list-tokens "$PBS_USER" --output-format json \ + | jq -e --arg t "$TOKEN_ID" 'any(.[]; .tokenid == $t)' >/dev/null +} + +# gen_token → SECRET on stdout of this function only (command substitution), nothing persisted. +gen_token() { + proxmox-backup-manager user generate-token "$PBS_USER" "$CID" \ + | sed -n 's/.*"value": "\([^"]*\)".*/\1/p' +} + +grant_both() { # dual-grant on the NAMESPACE ACL path (never /ns/ — §4a gotcha) + proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$PBS_USER" >&2 + proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup --auth-id "$TOKEN_ID" >&2 +} + +self_check() { # own-namespace list AS the new token; secret via env, never argv/stderr + PBS_PASSWORD="$1" proxmox-backup-client snapshot list --ns "$CID" \ + --repository "$TOKEN_ID@$REPO_HOST:$DS" >/dev/null +} + +rollback_token() { + proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true + proxmox-backup-manager acl update "/datastore/$DS/$CID" DatastoreBackup \ + --auth-id "$PBS_USER" --delete >&2 2>/dev/null || true +} + +if [ "$OP" = "provision" ]; then + if token_exists; then + err_json token_exists "token $TOKEN_ID already exists — use the reissue op (explicit re-key)" + fi +else # reissue: delete-token purges its ACLs; recreate + re-grant below + proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true + log "reissue: old token deleted (ACLs purged with it)" +fi + +SECRET=$(gen_token) +[ -n "$SECRET" ] || err_json internal "token generation returned no value" +grant_both + +if ! self_check "$SECRET"; then + # The spike's transient-403 note: one delete+regen+re-grant retry, then rollback + fail. + log "self-check failed — regenerating once (spike transient-403 note)" + proxmox-backup-manager user delete-token "$PBS_USER" "$CID" >&2 2>/dev/null || true + SECRET=$(gen_token) + [ -n "$SECRET" ] || { rollback_token; err_json internal "token regeneration returned no value"; } + grant_both + if ! self_check "$SECRET"; then + rollback_token + err_json self_check_failed "own-namespace list as $TOKEN_ID failed twice — rolled back" + fi +fi + +log "$OP ok: ns=$CID token=$TOKEN_ID (secret rides stdout only)" +jq -cn --arg tid "$TOKEN_ID" --arg sec "$SECRET" --arg fp "$PBS_FINGERPRINT" \ + --arg ds "$DS" --arg ns "$CID" \ + '{"status":"ok","token_id":$tid,"token_secret":$sec,"fingerprint":$fp,"datastore":$ds,"namespace":$ns}'