fix(pbs): Verify forces ignore-verified=false; restore-test source_tier from storage type
Live PBS runbook surfaced two gaps: (1) PBS verify defaults to ignore-verified=true and
SKIPS already-verified snapshots, so corruption after the first verify is never caught —
the agent's integrity check now POSTs ignore-verified=false to actually re-read+re-check.
(2) restore-test source_tier was hardcoded 'local'; now derived from the source storage
type ('pbs' for a PBS datastore). Adds a form-POST path to the PBS client.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -346,6 +346,21 @@ func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logge
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storageTier returns the restore-test source tier for a backup storage id: "pbs" when that
|
||||||
|
// storage is a PBS datastore, else "local". Best-effort (a lookup failure → "local").
|
||||||
|
func storageTier(ctx context.Context, px *proxmox.Client, storageID string) string {
|
||||||
|
stores, err := px.ListStorage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "local"
|
||||||
|
}
|
||||||
|
for _, s := range stores {
|
||||||
|
if s.Storage == storageID && s.Type == "pbs" {
|
||||||
|
return "pbs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "local"
|
||||||
|
}
|
||||||
|
|
||||||
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
|
// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
|
||||||
func readTrimmed(path string) (string, error) {
|
func readTrimmed(path string) (string, error) {
|
||||||
b, err := os.ReadFile(path)
|
b, err := os.ReadFile(path)
|
||||||
@@ -381,7 +396,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
|
|||||||
RestoreStorage: cfg.Backup.RestoreStorage,
|
RestoreStorage: cfg.Backup.RestoreStorage,
|
||||||
ScratchMin: min,
|
ScratchMin: min,
|
||||||
ScratchMax: max,
|
ScratchMax: max,
|
||||||
SourceTier: "local",
|
SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget),
|
||||||
},
|
},
|
||||||
Cadence: cadence,
|
Cadence: cadence,
|
||||||
Logger: logger,
|
Logger: logger,
|
||||||
@@ -665,7 +680,7 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
|
|||||||
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
||||||
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
||||||
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
||||||
ScratchMin: min, ScratchMax: max, SourceTier: "local",
|
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget),
|
||||||
})
|
})
|
||||||
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
||||||
if res.Skipped {
|
if res.Skipped {
|
||||||
|
|||||||
+24
-7
@@ -67,7 +67,12 @@ func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (str
|
|||||||
Data string `json:"data"`
|
Data string `json:"data"`
|
||||||
}
|
}
|
||||||
path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore))
|
path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore))
|
||||||
if err := c.do(ctx, http.MethodPost, path, &out); err != nil {
|
// ignore-verified=false → RE-verify even already-verified snapshots. This is what makes
|
||||||
|
// the check actually detect corruption (PBS's default ignore-verified=true skips them, so
|
||||||
|
// a chunk that rots after its first verify would never be re-checked). The cost is real
|
||||||
|
// re-read I/O; for a large datastore a future refinement is outdated-after-based scoping.
|
||||||
|
form := url.Values{"ignore-verified": {"false"}}
|
||||||
|
if err := c.post(ctx, path, form, &out); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return out.Data, nil
|
return out.Data, nil
|
||||||
@@ -175,25 +180,37 @@ func NodeFromUPID(upid string) string {
|
|||||||
return parts[1]
|
return parts[1]
|
||||||
}
|
}
|
||||||
|
|
||||||
// do performs a request, sets the token auth header, and decodes the JSON body into out. The
|
// post performs a form-encoded POST (PBS mutating ops take form params).
|
||||||
// auth header carries the secret and is NEVER logged.
|
func (c *Client) post(ctx context.Context, path string, form url.Values, out any) error {
|
||||||
|
return c.doBody(ctx, http.MethodPost, path, strings.NewReader(form.Encode()), "application/x-www-form-urlencoded", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// do performs a request (no body), sets the token auth header, and decodes JSON into out.
|
||||||
func (c *Client) do(ctx context.Context, method, path string, out any) error {
|
func (c *Client) do(ctx context.Context, method, path string, out any) error {
|
||||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, nil)
|
return c.doBody(ctx, method, path, nil, "", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// doBody is the shared request path. The auth header carries the secret and is NEVER logged.
|
||||||
|
func (c *Client) doBody(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if contentType != "" {
|
||||||
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
}
|
||||||
req.Header.Set("Authorization", c.authHeader)
|
req.Header.Set("Authorization", c.authHeader)
|
||||||
resp, err := c.http.Do(req)
|
resp, err := c.http.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("pbs: %s %s: %w", method, path, err)
|
return fmt.Errorf("pbs: %s %s: %w", method, path, err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(body))
|
return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(respBody))
|
||||||
}
|
}
|
||||||
if out != nil {
|
if out != nil {
|
||||||
if err := json.Unmarshal(body, out); err != nil {
|
if err := json.Unmarshal(respBody, out); err != nil {
|
||||||
return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err)
|
return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user