fix(pbs): Verify forces ignore-verified=false; restore-test source_tier from storage type
Live PBS runbook surfaced two gaps: (1) PBS verify defaults to ignore-verified=true and
SKIPS already-verified snapshots, so corruption after the first verify is never caught —
the agent's integrity check now POSTs ignore-verified=false to actually re-read+re-check.
(2) restore-test source_tier was hardcoded 'local'; now derived from the source storage
type ('pbs' for a PBS datastore). Adds a form-POST path to the PBS client.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -346,6 +346,21 @@ func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logge
|
||||
}
|
||||
}
|
||||
|
||||
// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
|
||||
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
|
||||
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
|
||||
stores, err := px.ListStorage(ctx)
|
||||
if err != nil {
|
||||
return "local"
|
||||
}
|
||||
for _, s := range stores {
|
||||
if s.Storage == storageID && s.Type == "pbs" {
|
||||
return "pbs"
|
||||
}
|
||||
}
|
||||
return "local"
|
||||
}
|
||||
|
||||
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
|
||||
func readTrimmed(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
@@ -381,7 +396,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
|
||||
RestoreStorage: cfg.Backup.RestoreStorage,
|
||||
ScratchMin: min,
|
||||
ScratchMax: max,
|
||||
SourceTier: "local",
|
||||
SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget),
|
||||
},
|
||||
Cadence: cadence,
|
||||
Logger: logger,
|
||||
@@ -665,7 +680,7 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
|
||||
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
||||
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
||||
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
||||
ScratchMin: min, ScratchMax: max, SourceTier: "local",
|
||||
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget),
|
||||
})
|
||||
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
||||
if res.Skipped {
|
||||
|
||||
+24
-7
@@ -67,7 +67,12 @@ func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (str
|
||||
Data string `json:"data"`
|
||||
}
|
||||
path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore))
|
||||
if err := c.do(ctx, http.MethodPost, path, &out); err != nil {
|
||||
// ignore-verified=false → RE-verify even already-verified snapshots. This is what makes
|
||||
// the check actually detect corruption (PBS's default ignore-verified=true skips them, so
|
||||
// a chunk that rots after its first verify would never be re-checked). The cost is real
|
||||
// re-read I/O; for a large datastore a future refinement is outdated-after-based scoping.
|
||||
form := url.Values{"ignore-verified": {"false"}}
|
||||
if err := c.post(ctx, path, form, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out.Data, nil
|
||||
@@ -175,25 +180,37 @@ func NodeFromUPID(upid string) string {
|
||||
return parts[1]
|
||||
}
|
||||
|
||||
// do performs a request, sets the token auth header, and decodes the JSON body into out. The
|
||||
// auth header carries the secret and is NEVER logged.
|
||||
// post performs a form-encoded POST (PBS mutating ops take form params).
|
||||
func (c *Client) post(ctx context.Context, path string, form url.Values, out any) error {
|
||||
return c.doBody(ctx, http.MethodPost, path, strings.NewReader(form.Encode()), "application/x-www-form-urlencoded", out)
|
||||
}
|
||||
|
||||
// do performs a request (no body), sets the token auth header, and decodes JSON into out.
|
||||
func (c *Client) do(ctx context.Context, method, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, nil)
|
||||
return c.doBody(ctx, method, path, nil, "", out)
|
||||
}
|
||||
|
||||
// doBody is the shared request path. The auth header carries the secret and is NEVER logged.
|
||||
func (c *Client) doBody(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
req.Header.Set("Authorization", c.authHeader)
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pbs: %s %s: %w", method, path, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(body))
|
||||
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(respBody))
|
||||
}
|
||||
if out != nil {
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
if err := json.Unmarshal(respBody, out); err != nil {
|
||||
return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user