v0.6.0: slice 6 Phase B — PBS offsite tier (verify + PBS-API client + reporting)

Spike-proven that backup/restore-to-PBS reuse Phase A unchanged; the only new code is
the verify capability, a small PBS-API client, and PBSSnapshot reporting.

- internal/pbs: fingerprint-pinned, token-authed PBS-API client (Verify/Snapshots/
  TaskStatus, node-from-UPID; secret read from /etc/pve/priv/storage/<id>.pw at runtime,
  never logged) + the verify maintenance loop (own cadence, default 6h, NOT gated/journaled,
  like the watchdog) + SnapshotStore.
- hub: PBSSnapshot filled (namespace/type/id/time/size/owner/protected/encrypted/
  verify_state/verify_upid); PBSReporter collector seam; cross-repo golden + bidirectional
  key-set tests; hub handler parses pbs_snapshots + logs a failed-verify WARN.
- backup: report the ACTUAL vzdump mode (parsed from the task log; PVE may downgrade
  snapshot->stop). proxmox.Storage.Username. config PBSVerifyCadence/secret-dir.
  --selftest=pbs-verify. Backup/restore-to-PBS unchanged.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

internal/pbs/verify.go

@@ -0,0 +1,127 @@
+package pbs
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+)
+
+// DefaultVerifyCadence is the verify maintenance interval — more frequent than the full
+// self-restore-test, because it's the cheap, key-free, ciphertext-level integrity check (§8).
+const DefaultVerifyCadence = 6 * time.Hour
+
+// Target is one PBS datastore to verify, with its client.
+type Target struct {
+	Datastore string
+	Client    *Client
+}
+
+// Targets resolves the current set of PBS datastores to verify (re-derived each cycle from
+// the PVE storage config — wired in main.go so pbs stays decoupled from how clients are built).
+type Targets func(ctx context.Context) ([]Target, error)
+
+// VerifyLoop is the verify maintenance loop (slice 6 Phase B). It runs on its OWN cadence and
+// is a reporting/maintenance task like the slice-5 watchdog — it does NOT go through the
+// reconcile gate/journal (it mutates no guest). Each cycle, per datastore: trigger a verify →
+// poll the task → re-list snapshots → record the per-snapshot verify-state for the report.
+type VerifyLoop struct {
+	targets Targets
+	store   *SnapshotStore
+	cadence time.Duration
+	logger  *slog.Logger
+}
+
+// VerifyLoopOptions configures a VerifyLoop.
+type VerifyLoopOptions struct {
+	Targets Targets
+	Store   *SnapshotStore
+	Cadence time.Duration // 0 → default 6h; negative → disabled
+	Logger  *slog.Logger
+}
+
+// NewVerifyLoop builds a VerifyLoop.
+func NewVerifyLoop(opts VerifyLoopOptions) *VerifyLoop {
+	logger := opts.Logger
+	if logger == nil {
+		logger = slog.Default()
+	}
+	cadence := opts.Cadence
+	if cadence == 0 {
+		cadence = DefaultVerifyCadence
+	}
+	return &VerifyLoop{targets: opts.Targets, store: opts.Store, cadence: cadence, logger: logger}
+}
+
+// Run verifies on the cadence until ctx is cancelled. It does an immediate first pass (so a
+// freshly-started agent reports snapshot inventory + verify-state promptly), then on each
+// tick. A negative cadence (or nil targets/store) disables it. Returns nil on cancellation.
+func (l *VerifyLoop) Run(ctx context.Context) error {
+	if l.cadence < 0 || l.targets == nil || l.store == nil {
+		l.logger.Info("pbs: verify loop disabled")
+		<-ctx.Done()
+		return nil
+	}
+	l.logger.Info("pbs: verify loop starting", "cadence", l.cadence)
+	l.tick(ctx) // immediate inventory + verify
+	t := time.NewTicker(l.cadence)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			l.logger.Info("pbs: verify loop shutting down", "reason", ctx.Err())
+			return nil
+		case <-t.C:
+			l.tick(ctx)
+		}
+	}
+}
+
+// RunOnce performs a single synchronous verify+list pass over all targets (used by the
+// selftest harness and the live runbook). Same work as one cadence tick.
+func (l *VerifyLoop) RunOnce(ctx context.Context) { l.tick(ctx) }
+
+// tick verifies + re-lists each target datastore once. Deterministic enough to drive directly
+// in tests. A per-target error is logged and skipped (other datastores still report).
+func (l *VerifyLoop) tick(ctx context.Context) {
+	targets, err := l.targets(ctx)
+	if err != nil {
+		l.logger.Warn("pbs: verify loop could not resolve targets; skipping", "err", err)
+		return
+	}
+	for _, t := range targets {
+		l.verifyOne(ctx, t)
+	}
+}
+
+// verifyOne triggers a verify, waits for it, then re-lists + records the snapshots' state.
+func (l *VerifyLoop) verifyOne(ctx context.Context, t Target) {
+	if upid, err := t.Client.Verify(ctx, t.Datastore); err != nil {
+		// Verify-trigger failure is non-fatal: still re-list so we report current state.
+		l.logger.Warn("pbs: verify trigger failed; reporting current snapshot state", "datastore", t.Datastore, "err", err)
+	} else if err := t.Client.WaitVerify(ctx, upid, 2*time.Second, 30*time.Minute); err != nil {
+		l.logger.Warn("pbs: verify task wait failed; reporting current snapshot state", "datastore", t.Datastore, "err", err)
+	}
+
+	snaps, err := t.Client.Snapshots(ctx, t.Datastore)
+	if err != nil {
+		l.logger.Warn("pbs: snapshot list failed", "datastore", t.Datastore, "err", err)
+		return
+	}
+	out := make([]hub.PBSSnapshot, 0, len(snaps))
+	failed := 0
+	for _, s := range snaps {
+		h := s.ToHub()
+		if h.VerifyState == VerifyFailed {
+			failed++
+		}
+		out = append(out, h)
+	}
+	l.store.Record(t.Datastore, out)
+	if failed > 0 {
+		l.logger.Error("pbs: datastore has FAILED-verify snapshots", "datastore", t.Datastore, "failed", failed, "total", len(out))
+	} else {
+		l.logger.Info("pbs: verify cycle complete", "datastore", t.Datastore, "snapshots", len(out))
+	}
+}