v0.6.0: slice 6 Phase B — PBS offsite tier (verify + PBS-API client + reporting)

Spike-proven that backup/restore-to-PBS reuse Phase A unchanged; the only new code is the verify capability, a small PBS-API client, and PBSSnapshot reporting. - internal/pbs: fingerprint-pinned, token-authed PBS-API client (Verify/Snapshots/ TaskStatus, node-from-UPID; secret read from /etc/pve/priv/storage/<id>.pw at runtime, never logged) + the verify maintenance loop (own cadence, default 6h, NOT gated/journaled, like the watchdog) + SnapshotStore. - hub: PBSSnapshot filled (namespace/type/id/time/size/owner/protected/encrypted/ verify_state/verify_upid); PBSReporter collector seam; cross-repo golden + bidirectional key-set tests; hub handler parses pbs_snapshots + logs a failed-verify WARN. - backup: report the ACTUAL vzdump mode (parsed from the task log; PVE may downgrade snapshot->stop). proxmox.Storage.Username. config PBSVerifyCadence/secret-dir. --selftest=pbs-verify. Backup/restore-to-PBS unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 16:53:04 +02:00
parent faba8e4ff7
commit 766500dfc3
20 changed files with 1082 additions and 37 deletions
@@ -25,6 +25,7 @@ import (
 	"gitea.dooplex.hu/admin/felhom-agent/internal/config"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
 	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
@@ -32,7 +33,7 @@ import (

 // version is the agent version. Overridable at build time with
 // -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
-var version = "0.6.0-rc1"
+var version = "0.6.0"

 func main() {
 	var (
@@ -44,7 +45,7 @@ func main() {
 		showVersion bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup)")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records")
 	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup")
 	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
 	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test: the backup volid to restore (default: newest on the configured local target)")
@@ -83,6 +84,8 @@ func main() {
 		os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
 	case "restore-test":
 		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
+	case "pbs-verify":
+		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
 	}
 }

@@ -189,7 +192,10 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	// latest restore-test result; the collector reads it via the BackupReporter /
 	// RestoreTestReporter seams; the cadence scheduler writes it.
 	backupStore := backup.NewStore()
-	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, cfg.Hub.HostID, version, logger)
+	// PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes it; the
+	// collector reads it via the PBSReporter seam.
+	pbsStore := pbs.NewSnapshotStore()
+	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsStore, cfg.Hub.HostID, version, logger)
 	loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
 	interval := time.Duration(hcfg.PollSeconds) * time.Second

@@ -276,19 +282,31 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	// OR the scratch band / restore storage is misconfigured — the daemon still runs.
 	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)

-	// Run reconcile, the hub loop, the storage watchdog, and the restore-test scheduler
-	// concurrently; any one returning ends the daemon (ctx cancellation tears down the rest).
-	errc := make(chan error, 4)
+	// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
+	// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
+	// verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs
+	// storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing.
+	pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
+		Targets: pbsTargetsFromPVE(cfg, px, logger),
+		Store:   pbsStore,
+		Cadence: cfg.Backup.PBSVerifyCadence(),
+		Logger:  logger,
+	})
+
+	// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, and the
+	// PBS verify loop concurrently; any one returning ends the daemon (ctx cancel tears down rest).
+	errc := make(chan error, 5)
 	go func() { errc <- engine.Run(ctx, interval) }()
 	go func() { errc <- loop.Run(ctx) }()
 	go func() { errc <- watchdog.Run(ctx) }()
 	go func() { errc <- scheduler.Run(ctx) }()
+	go func() { errc <- pbsLoop.Run(ctx) }()

 	err = <-errc
-	stop() // tear down the siblings on the first exit
-	<-errc // wait for the second
-	<-errc // wait for the third
-	<-errc // wait for the fourth
+	stop()                   // tear down the siblings on the first exit
+	for i := 0; i < 4; i++ { // wait for the other four
+		<-errc
+	}
 	if err != nil && err != context.Canceled {
 		logger.Error("daemon: exited with error", "err", err)
 		return 1
@@ -296,6 +314,51 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	return 0
 }

+// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs
+// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each
+// (token id from the storage `username`, secret read from <PBSSecretDir>/<id>.pw). A storage
+// whose secret/client can't be built is skipped with a warning (the loop still verifies the
+// rest). The secret is read at runtime, never logged.
+func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets {
+	return func(ctx context.Context) ([]pbs.Target, error) {
+		stores, err := px.ListStorage(ctx)
+		if err != nil {
+			return nil, err
+		}
+		var targets []pbs.Target
+		for _, s := range stores {
+			if s.Type != "pbs" {
+				continue
+			}
+			secret, err := readTrimmed(cfg.Backup.PBSSecretPath(s.Storage))
+			if err != nil {
+				logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err)
+				continue
+			}
+			c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret})
+			if err != nil {
+				logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err)
+				continue
+			}
+			targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c})
+		}
+		return targets, nil
+	}
+}
+
+// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret).
+func readTrimmed(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	s := strings.TrimSpace(string(b))
+	if s == "" {
+		return "", fmt.Errorf("empty file %s", path)
+	}
+	return s, nil
+}
+
 // buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
 // disables the cadence (returns a scheduler that just waits) when the cadence is off or the
 // scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
@@ -386,7 +449,7 @@ func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger)
 		return 1
 	}
 	observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
-	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, cfg.Hub.HostID, version, logger)
+	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, nil, cfg.Hub.HostID, version, logger)

 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
@@ -617,6 +680,56 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
 	return 0
 }

+// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
+// path), then lists + prints the resulting PBSSnapshot records (verify-state included).
+// Standalone on the host. Covers the runbook's (c) verify and (d) list.
+func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+
+	fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version)
+	targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "  [FAIL] discover pbs storages:", err)
+		return 1
+	}
+	if len(targets) == 0 {
+		fmt.Println("  no pbs storages configured on this host")
+		return 0
+	}
+	store := pbs.NewSnapshotStore()
+	loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{
+		Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil },
+		Store:   store,
+		Logger:  logger,
+	})
+	// One synchronous verify+list pass over all datastores.
+	loop.RunOnce(ctx)
+	snaps := store.PBSSnapshots(ctx)
+	printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps)
+	failed := 0
+	for _, s := range snaps {
+		if s.VerifyState == pbs.VerifyFailed {
+			failed++
+		}
+	}
+	if failed > 0 {
+		fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed)
+		return 1
+	}
+	fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets))
+	return 0
+}
+
 // printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
 func printJSON(label string, v any) {
 	if b, err := json.MarshalIndent(v, "  ", "  "); err == nil {
@@ -916,8 +1029,10 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "backup"
 	case "restore-test":
 		f.mode = "restore-test"
+	case "pbs-verify":
+		f.mode = "pbs-verify"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify)", v)
 	}
 	return nil
 }