pbs: namespace-aware client for per-customer offsite tenancy (S4)

Phase-1 live probe (felhom-hetzner) proved backup/restore/list/isolation over
the tunnel with a per-customer DatastoreBackup token, but the agent's PBS client
was namespace-unaware: Snapshots hit the datastore root (403 for a scoped token)
and Verify was whole-datastore (needs Datastore.Verify ~ admin). Operator-
approved fix.

- pbs.Config.Namespace + Client.namespace; Snapshots appends ?ns=; Verify sends
  ns= (ns-scoped verify works with DatastoreBackup on the own ns — no admin
  widening, Phase-1 confirmed). Root-ns clients unchanged (whole-datastore).
- proxmox.Storage.Namespace (parsed from /storage `namespace`).
- pbsTargetsFromPVE threads s.Namespace into the client.

Confirmed tenant ACL: DatastoreBackup on /datastore/felhom-offsite/<ns> (NOT
/ns/<ns>) to BOTH felhom@pbs (user) AND felhom@pbs!<ns> (token) — PBS privsep =
intersection; isolation holds (cross-ns 403 proven). TestClient_NamespaceScoping
red-proofed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

CHANGELOG.md

@@ -1,3 +1,26 @@
+## v0.67.0 — S4: namespace-aware PBS client (per-customer offsite tenancy) (2026-07-04)
+
+Phase-1 live probe on felhom-hetzner proved the offsite tenancy path (backup/restore/list/isolation
+all green over the tunnel with a per-customer DatastoreBackup token) but surfaced that the agent's
+PBS client was **namespace-unaware**: `Snapshots` hit the datastore root (403 for a scoped token)
+and `Verify` was whole-datastore (needs Datastore.Verify ~ admin). Operator-approved small change to
+make the client namespace-scoped so a properly-isolated token services its own tenant.
+
+- **`internal/pbs`**: `Config.Namespace` (+ `Client.namespace`). `Snapshots` appends `?ns=<ns>`
+  (lists ONLY the tenant's namespace); `Verify` sends `ns=<ns>` (verifies ONLY that namespace —
+  Phase-1-confirmed to work with a **DatastoreBackup** token on its own ns, no Datastore.Verify /
+  admin widening). Root-ns clients (Namespace="") are unchanged → whole-datastore (the DooPlex
+  `felhom-pbs` n100 path). Test `TestClient_NamespaceScoping` pins both (red-proofed).
+- **`internal/proxmox`**: `Storage.Namespace` (parsed from the PVE `/storage` config key `namespace`).
+- **`cmd/felhom-agent`**: `pbsTargetsFromPVE` threads `s.Namespace` into the PBS client, so a PBS
+  storage configured with a namespace is verified/reported scoped to it automatically.
+
+**Confirmed minimal tenant ACL (recorded live 2026-07-04, felhom-hetzner):** `DatastoreBackup` on
+`/datastore/felhom-offsite/<ns>` (the namespace path — NOT `/ns/<ns>`) granted to **BOTH** the user
+`felhom@pbs` **and** the token `felhom@pbs!<ns>` — PBS privsep tokens = intersection(user, token),
+so both are required; isolation holds because each token's ACL is only its own ns (cross-ns
+list/backup → 403, proven). No token exceeds DatastoreBackup; no admin on the endpoint for the box.
+
 ## v0.66.0 — S4 agent half: endpoint v4-pin + re-resolve watchdog + FELHOM_WG Critical flips (2026-07-04)
 
 The two agent items S4 needs before offsite backups ride the tunnel (the tenancy + storage weight is

cmd/felhom-agent/main.go

@@ -753,7 +753,7 @@ func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logge
 				logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
 				continue
 			}
-			c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret})
+			c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret, Namespace: s.Namespace})
 			if err != nil {
 				logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
 				continue

internal/pbs/client.go

@@ -17,6 +17,7 @@ type Client struct {
 	base       string // https://<server>:<port>/api2/json
 	authHeader string // "PBSAPIToken=<tokenid>:<secret>" — SECRET; never logged
 	http       *http.Client
+	namespace  string // "" = root ns (whole-datastore); set = per-customer tenant scope (S4)
 }
 
 // Config builds a Client. Secret is read by the caller from /etc/pve/priv/storage/<id>.pw at
@@ -27,6 +28,7 @@ type Config struct {
 	Fingerprint string // SHA-256 of the PBS leaf cert (colons optional)
 	TokenID     string // e.g. "felhom@pbs!n100" (from storage.cfg `username`)
 	Secret      string // token secret (from <id>.pw)
+	Namespace   string // PBS namespace (from storage.cfg `namespace`); "" = root. S4 per-customer tenancy.
 	Timeout     time.Duration
 }
 
@@ -50,6 +52,7 @@ func NewClient(cfg Config) (*Client, error) {
 	return &Client{
 		base:       fmt.Sprintf("https://%s:%d/api2/json", cfg.Server, port),
 		authHeader: "PBSAPIToken=" + cfg.TokenID + ":" + cfg.Secret,
+		namespace:  cfg.Namespace,
 		http: &http.Client{
 			Timeout:   timeout,
 			Transport: &http.Transport{TLSClientConfig: tlsCfg},
@@ -57,11 +60,12 @@ func NewClient(cfg Config) (*Client, error) {
 	}, nil
 }
 
-// Verify triggers a datastore verify (POST /admin/datastore/<ds>/verify) and returns the
+// Verify triggers a verify (POST /admin/datastore/<ds>/verify) and returns the task UPID; the
-// task UPID. With no snapshots it verifies the whole datastore; the cheap, key-free,
+// cheap, key-free, ciphertext-level integrity check (doc 03 §8). When the client is namespace-
-// ciphertext-level integrity check (doc 03 §8). Needs the token's Datastore.Verify (in
+// scoped (S4 per-customer tenancy) the verify is confined to that namespace (`ns=`), which a
-// DatastoreAdmin). Per-snapshot scoping is a future refinement; whole-datastore is the spike-
+// DatastoreBackup token can trigger on its OWN namespace — no Datastore.Verify / admin widening
-// proven path.
+// (Phase-1 confirmed live 2026-07-04). Root-ns (unscoped) clients verify the whole datastore as
+// before (needs Datastore.Verify, e.g. the DooPlex felhom-pbs n100 token).
 func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (string, error) {
 	var out struct {
 		Data string `json:"data"`
@@ -72,6 +76,9 @@ func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (str
 	// a chunk that rots after its first verify would never be re-checked). The cost is real
 	// re-read I/O; for a large datastore a future refinement is outdated-after-based scoping.
 	form := url.Values{"ignore-verified": {"false"}}
+	if c.namespace != "" {
+		form.Set("ns", c.namespace)
+	}
 	if err := c.post(ctx, path, form, &out); err != nil {
 		return "", err
 	}
@@ -98,12 +105,18 @@ type Snapshot struct {
 	} `json:"files"`
 }
 
-// Snapshots lists the datastore's snapshots (incl. the verification field).
+// Snapshots lists the datastore's snapshots (incl. the verification field). A namespace-scoped
+// client (S4) lists ONLY its namespace (`?ns=`) — the unscoped call targets the datastore root,
+// which a per-customer DatastoreBackup token cannot read (Phase-1: 403 without ns). Root-ns
+// clients list the root namespace as before.
 func (c *Client) Snapshots(ctx context.Context, datastore string) ([]Snapshot, error) {
 	var out struct {
 		Data []Snapshot `json:"data"`
 	}
 	path := fmt.Sprintf("/admin/datastore/%s/snapshots", url.PathEscape(datastore))
+	if c.namespace != "" {
+		path += "?ns=" + url.QueryEscape(c.namespace)
+	}
 	if err := c.do(ctx, http.MethodGet, path, &out); err != nil {
 		return nil, err
 	}

internal/pbs/client_test.go

@@ -159,3 +159,64 @@ func TestNormalizeFingerprint(t *testing.T) {
 		t.Errorf("normalize = %q err=%v (want lowercased, colons stripped)", got, err)
 	}
 }
+
+// TestClient_NamespaceScoping pins the S4 per-customer tenancy behavior: a namespace-scoped client
+// lists ONLY its namespace (snapshots ?ns=) and verifies ONLY its namespace (verify ns=), so a
+// DatastoreBackup token never touches the datastore root; a root client (no namespace) sends
+// neither — whole-datastore behavior preserved (the DooPlex felhom-pbs path). Red-proof: drop the
+// `if c.namespace != ""` guard in Snapshots/Verify and the scoped assertions fail.
+func TestClient_NamespaceScoping(t *testing.T) {
+	var gotSnapQuery, gotVerifyNS string
+	ts, fp := newPBSTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "/snapshots"):
+			gotSnapQuery = r.URL.RawQuery
+			w.Write([]byte(`{"data":[]}`))
+		case strings.Contains(r.URL.Path, "/verify"):
+			r.ParseForm()
+			gotVerifyNS = r.PostFormValue("ns")
+			w.Write([]byte(`{"data":"UPID:node:1:2:3:4:verify:ds:felhom@pbs!demo:"}`))
+		default:
+			w.Write([]byte(`{"data":[]}`))
+		}
+	})
+	host, port := hostPort(t, ts.URL)
+	ctx := context.Background()
+
+	// Namespace-scoped tenant client.
+	scoped, err := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "felhom@pbs!demo-felhom-01", Secret: "s", Namespace: "demo-felhom-01"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := scoped.Snapshots(ctx, "felhom-offsite"); err != nil {
+		t.Fatal(err)
+	}
+	if gotSnapQuery != "ns=demo-felhom-01" {
+		t.Errorf("scoped snapshots query = %q, want ns=demo-felhom-01", gotSnapQuery)
+	}
+	if _, err := scoped.Verify(ctx, "felhom-offsite"); err != nil {
+		t.Fatal(err)
+	}
+	if gotVerifyNS != "demo-felhom-01" {
+		t.Errorf("scoped verify ns = %q, want demo-felhom-01", gotVerifyNS)
+	}
+
+	// Root client: no namespace → whole-datastore, no ns on either call.
+	gotSnapQuery, gotVerifyNS = "SENTINEL", "SENTINEL"
+	root, err := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "felhom@pbs!n100", Secret: "s"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := root.Snapshots(ctx, "felhom-spike"); err != nil {
+		t.Fatal(err)
+	}
+	if gotSnapQuery != "" {
+		t.Errorf("root snapshots query = %q, want empty (no ns → whole datastore)", gotSnapQuery)
+	}
+	if _, err := root.Verify(ctx, "felhom-spike"); err != nil {
+		t.Fatal(err)
+	}
+	if gotVerifyNS != "" {
+		t.Errorf("root verify ns = %q, want empty (no ns → whole datastore)", gotVerifyNS)
+	}
+}

internal/proxmox/types.go

@@ -218,6 +218,7 @@ type Storage struct {
 	Datastore   string `json:"datastore,omitempty"`   // pbs datastore name
 	Fingerprint string `json:"fingerprint,omitempty"` // pbs server cert fingerprint
 	Username    string `json:"username,omitempty"`    // pbs auth id, e.g. "felhom@pbs!n100"
+	Namespace   string `json:"namespace,omitempty"`   // pbs namespace ("" = root; per-customer tenancy = S4)
 	VGName      string `json:"vgname,omitempty"`      // lvm/lvmthin volume group
 	ThinPool    string `json:"thinpool,omitempty"`    // lvmthin pool LV name
 }