fix(pbs): Verify forces ignore-verified=false; restore-test source_tier from storage type

Live PBS runbook surfaced two gaps: (1) PBS verify defaults to ignore-verified=true and
SKIPS already-verified snapshots, so corruption after the first verify is never caught —
the agent's integrity check now POSTs ignore-verified=false to actually re-read+re-check.
(2) restore-test source_tier was hardcoded 'local'; now derived from the source storage
type ('pbs' for a PBS datastore). Adds a form-POST path to the PBS client.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 17:00:21 +02:00
parent 766500dfc3
commit dcd8a8eff4
2 changed files with 41 additions and 9 deletions
+17 -2
View File
@@ -346,6 +346,21 @@ func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logge
}
}
// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
stores, err := px.ListStorage(ctx)
if err != nil {
return "local"
}
for _, s := range stores {
if s.Storage == storageID && s.Type == "pbs" {
return "pbs"
}
}
return "local"
}
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
func readTrimmed(path string) (string, error) {
b, err := os.ReadFile(path)
@@ -381,7 +396,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
RestoreStorage: cfg.Backup.RestoreStorage,
ScratchMin: min,
ScratchMax: max,
SourceTier: "local",
SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget),
},
Cadence: cadence,
Logger: logger,
@@ -665,7 +680,7 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
ScratchMin: min, ScratchMax: max, SourceTier: "local",
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget),
})
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
if res.Skipped {
+24 -7
View File
@@ -67,7 +67,12 @@ func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (str
Data string `json:"data"`
}
path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore))
if err := c.do(ctx, http.MethodPost, path, &out); err != nil {
// ignore-verified=false → RE-verify even already-verified snapshots. This is what makes
// the check actually detect corruption (PBS's default ignore-verified=true skips them, so
// a chunk that rots after its first verify would never be re-checked). The cost is real
// re-read I/O; for a large datastore a future refinement is outdated-after-based scoping.
form := url.Values{"ignore-verified": {"false"}}
if err := c.post(ctx, path, form, &out); err != nil {
return "", err
}
return out.Data, nil
@@ -175,25 +180,37 @@ func NodeFromUPID(upid string) string {
return parts[1]
}
// do performs a request, sets the token auth header, and decodes the JSON body into out. The
// auth header carries the secret and is NEVER logged.
// post performs a form-encoded POST (PBS mutating ops take form params).
func (c *Client) post(ctx context.Context, path string, form url.Values, out any) error {
return c.doBody(ctx, http.MethodPost, path, strings.NewReader(form.Encode()), "application/x-www-form-urlencoded", out)
}
// do performs a request (no body), sets the token auth header, and decodes JSON into out.
func (c *Client) do(ctx context.Context, method, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, method, c.base+path, nil)
return c.doBody(ctx, method, path, nil, "", out)
}
// doBody is the shared request path. The auth header carries the secret and is NEVER logged.
func (c *Client) doBody(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error {
req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
if err != nil {
return err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
req.Header.Set("Authorization", c.authHeader)
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("pbs: %s %s: %w", method, path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(body))
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(respBody))
}
if out != nil {
if err := json.Unmarshal(body, out); err != nil {
if err := json.Unmarshal(respBody, out); err != nil {
return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err)
}
}