v0.6.0: slice 6 Phase B — PBS offsite tier (verify + PBS-API client + reporting)

Spike-proven that backup/restore-to-PBS reuse Phase A unchanged; the only new code is the verify capability, a small PBS-API client, and PBSSnapshot reporting. - internal/pbs: fingerprint-pinned, token-authed PBS-API client (Verify/Snapshots/ TaskStatus, node-from-UPID; secret read from /etc/pve/priv/storage/<id>.pw at runtime, never logged) + the verify maintenance loop (own cadence, default 6h, NOT gated/journaled, like the watchdog) + SnapshotStore. - hub: PBSSnapshot filled (namespace/type/id/time/size/owner/protected/encrypted/ verify_state/verify_upid); PBSReporter collector seam; cross-repo golden + bidirectional key-set tests; hub handler parses pbs_snapshots + logs a failed-verify WARN. - backup: report the ACTUAL vzdump mode (parsed from the task log; PVE may downgrade snapshot->stop). proxmox.Storage.Username. config PBSVerifyCadence/secret-dir. --selftest=pbs-verify. Backup/restore-to-PBS unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 16:53:04 +02:00
parent faba8e4ff7
commit 766500dfc3
20 changed files with 1082 additions and 37 deletions
@@ -0,0 +1,209 @@
+package pbs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// Client is the PBS-API client for ONE PBS server. Construct with NewClient. It is pure (no
+// logger — it must never log the token secret); callers log around it.
+type Client struct {
+	base       string // https://<server>:<port>/api2/json
+	authHeader string // "PBSAPIToken=<tokenid>:<secret>" — SECRET; never logged
+	http       *http.Client
+}
+
+// Config builds a Client. Secret is read by the caller from /etc/pve/priv/storage/<id>.pw at
+// runtime (referenced by location, never committed).
+type Config struct {
+	Server      string // PBS host (no scheme), e.g. "192.168.0.180"
+	Port        int    // default 8007
+	Fingerprint string // SHA-256 of the PBS leaf cert (colons optional)
+	TokenID     string // e.g. "felhom@pbs!n100" (from storage.cfg `username`)
+	Secret      string // token secret (from <id>.pw)
+	Timeout     time.Duration
+}
+
+// NewClient builds a fingerprint-pinned, token-authed PBS client.
+func NewClient(cfg Config) (*Client, error) {
+	if cfg.Server == "" || cfg.Fingerprint == "" || cfg.TokenID == "" || cfg.Secret == "" {
+		return nil, fmt.Errorf("pbs: NewClient needs server, fingerprint, tokenid and secret")
+	}
+	tlsCfg, err := pinnedTLS(cfg.Fingerprint)
+	if err != nil {
+		return nil, err
+	}
+	port := cfg.Port
+	if port == 0 {
+		port = 8007
+	}
+	timeout := cfg.Timeout
+	if timeout == 0 {
+		timeout = 30 * time.Second
+	}
+	return &Client{
+		base:       fmt.Sprintf("https://%s:%d/api2/json", cfg.Server, port),
+		authHeader: "PBSAPIToken=" + cfg.TokenID + ":" + cfg.Secret,
+		http: &http.Client{
+			Timeout:   timeout,
+			Transport: &http.Transport{TLSClientConfig: tlsCfg},
+		},
+	}, nil
+}
+
+// Verify triggers a datastore verify (POST /admin/datastore/<ds>/verify) and returns the
+// task UPID. With no snapshots it verifies the whole datastore; the cheap, key-free,
+// ciphertext-level integrity check (doc 03 §8). Needs the token's Datastore.Verify (in
+// DatastoreAdmin). Per-snapshot scoping is a future refinement; whole-datastore is the spike-
+// proven path.
+func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (string, error) {
+	var out struct {
+		Data string `json:"data"`
+	}
+	path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore))
+	if err := c.do(ctx, http.MethodPost, path, &out); err != nil {
+		return "", err
+	}
+	return out.Data, nil
+}
+
+// Snapshot is one PBS snapshot as the API returns it (GET /admin/datastore/<ds>/snapshots).
+type Snapshot struct {
+	BackupType   string `json:"backup-type"` // ct | vm
+	BackupID     string `json:"backup-id"`
+	BackupTime   int64  `json:"backup-time"` // epoch seconds
+	Size         int64  `json:"size"`
+	Owner        string `json:"owner"`
+	Protected    bool   `json:"protected"`
+	Namespace    string `json:"ns"` // "" = root namespace
+	Verification *struct {
+		State string `json:"state"` // ok | failed
+		UPID  string `json:"upid"`
+	} `json:"verification"`
+	Files []struct {
+		Filename  string `json:"filename"`
+		CryptMode string `json:"crypt-mode"` // encrypt | sign-only | (none)
+		Size      int64  `json:"size"`
+	} `json:"files"`
+}
+
+// Snapshots lists the datastore's snapshots (incl. the verification field).
+func (c *Client) Snapshots(ctx context.Context, datastore string) ([]Snapshot, error) {
+	var out struct {
+		Data []Snapshot `json:"data"`
+	}
+	path := fmt.Sprintf("/admin/datastore/%s/snapshots", url.PathEscape(datastore))
+	if err := c.do(ctx, http.MethodGet, path, &out); err != nil {
+		return nil, err
+	}
+	return out.Data, nil
+}
+
+// TaskStatus is the subset of a PBS task status we need.
+type TaskStatus struct {
+	Status     string `json:"status"`     // running | stopped
+	ExitStatus string `json:"exitstatus"` // present once stopped ("OK" or an error)
+	Node       string `json:"node"`
+}
+
+// Running reports whether the task is still executing.
+func (t TaskStatus) Running() bool { return t.Status == "running" }
+
+// OK reports whether the task stopped successfully.
+func (t TaskStatus) OK() bool { return t.Status == "stopped" && t.ExitStatus == "OK" }
+
+// TaskStatus polls a verify task. The PBS node name is extracted from the UPID — querying the
+// wrong node (e.g. "localhost") returns exitstatus "unknown" (the spike B4 gotcha).
+func (c *Client) TaskStatus(ctx context.Context, upid string) (TaskStatus, error) {
+	node := NodeFromUPID(upid)
+	if node == "" {
+		return TaskStatus{}, fmt.Errorf("pbs: cannot extract node from UPID %q", upid)
+	}
+	var out struct {
+		Data TaskStatus `json:"data"`
+	}
+	path := fmt.Sprintf("/nodes/%s/tasks/%s/status", url.PathEscape(node), url.PathEscape(upid))
+	if err := c.do(ctx, http.MethodGet, path, &out); err != nil {
+		return TaskStatus{}, err
+	}
+	return out.Data, nil
+}
+
+// WaitVerify polls a verify task until it stops or ctx/timeout elapses (best-effort — a poll
+// failure returns the error; the caller re-lists snapshots for the authoritative state).
+func (c *Client) WaitVerify(ctx context.Context, upid string, poll, timeout time.Duration) error {
+	if poll <= 0 {
+		poll = 2 * time.Second
+	}
+	if timeout <= 0 {
+		timeout = 10 * time.Minute
+	}
+	deadline := time.Now().Add(timeout)
+	t := time.NewTicker(poll)
+	defer t.Stop()
+	for {
+		st, err := c.TaskStatus(ctx, upid)
+		if err != nil {
+			return err
+		}
+		if !st.Running() {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("pbs: verify task %s did not finish within %s", upid, timeout)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+
+// NodeFromUPID extracts the node from a PBS/PVE UPID ("UPID:<node>:<pid>:...").
+func NodeFromUPID(upid string) string {
+	parts := strings.Split(upid, ":")
+	if len(parts) < 2 || parts[0] != "UPID" {
+		return ""
+	}
+	return parts[1]
+}
+
+// do performs a request, sets the token auth header, and decodes the JSON body into out. The
+// auth header carries the secret and is NEVER logged.
+func (c *Client) do(ctx context.Context, method, path string, out any) error {
+	req, err := http.NewRequestWithContext(ctx, method, c.base+path, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", c.authHeader)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("pbs: %s %s: %w", method, path, err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(body))
+	}
+	if out != nil {
+		if err := json.Unmarshal(body, out); err != nil {
+			return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err)
+		}
+	}
+	return nil
+}
+
+func trimBody(b []byte) string {
+	s := strings.TrimSpace(string(b))
+	if len(s) > 300 {
+		return s[:300] + "…"
+	}
+	return s
+}