From 766500dfc3b3b2f518a20c4a907f8669106b3b6e Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 9 Jun 2026 16:53:04 +0200 Subject: [PATCH] =?UTF-8?q?v0.6.0:=20slice=206=20Phase=20B=20=E2=80=94=20P?= =?UTF-8?q?BS=20offsite=20tier=20(verify=20+=20PBS-API=20client=20+=20repo?= =?UTF-8?q?rting)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spike-proven that backup/restore-to-PBS reuse Phase A unchanged; the only new code is the verify capability, a small PBS-API client, and PBSSnapshot reporting. - internal/pbs: fingerprint-pinned, token-authed PBS-API client (Verify/Snapshots/ TaskStatus, node-from-UPID; secret read from /etc/pve/priv/storage/.pw at runtime, never logged) + the verify maintenance loop (own cadence, default 6h, NOT gated/journaled, like the watchdog) + SnapshotStore. - hub: PBSSnapshot filled (namespace/type/id/time/size/owner/protected/encrypted/ verify_state/verify_upid); PBSReporter collector seam; cross-repo golden + bidirectional key-set tests; hub handler parses pbs_snapshots + logs a failed-verify WARN. - backup: report the ACTUAL vzdump mode (parsed from the task log; PVE may downgrade snapshot->stop). proxmox.Storage.Username. config PBSVerifyCadence/secret-dir. --selftest=pbs-verify. Backup/restore-to-PBS unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 42 ++++ CLAUDE.md | 6 +- cmd/felhom-agent/main.go | 139 +++++++++++- configs/agent.example.json | 4 +- internal/backup/backup_test.go | 21 ++ internal/backup/runner.go | 22 ++ internal/config/config.go | 29 +++ internal/hub/collect.go | 29 ++- internal/hub/collect_test.go | 12 +- internal/hub/contract_test.go | 13 +- internal/hub/report.go | 34 ++- internal/hub/testdata/host-report.golden.json | 15 +- internal/pbs/client.go | 209 ++++++++++++++++++ internal/pbs/client_test.go | 161 ++++++++++++++ internal/pbs/doc.go | 23 ++ internal/pbs/pin.go | 48 ++++ internal/pbs/report.go | 84 +++++++ internal/pbs/verify.go | 127 +++++++++++ internal/pbs/verify_test.go | 100 +++++++++ internal/proxmox/types.go | 1 + 20 files changed, 1082 insertions(+), 37 deletions(-) create mode 100644 internal/pbs/client.go create mode 100644 internal/pbs/client_test.go create mode 100644 internal/pbs/doc.go create mode 100644 internal/pbs/pin.go create mode 100644 internal/pbs/report.go create mode 100644 internal/pbs/verify.go create mode 100644 internal/pbs/verify_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1475633..c3ee49f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,48 @@ All notable changes to **felhom-agent** are recorded here. Update on every code change that gets pushed. +## v0.6.0 — slice 6 Phase B: PBS offsite tier (verify + PBS-API client + reporting) (2026-06-09) + +Completes slice 6. The PBS spike (felhom.eu phase5-pbs-spike-findings.md) proved backup-to-PBS +and restore-from-PBS reuse Phase A UNCHANGED (PBS is just a storage target + a volid), and the +operator token needs no widening. So the only new agent code is the **verify capability + a +small PBS-API client + PBSSnapshot reporting**. Escrow + host-loss DR stay slices 7/10. + +### Added +- **`internal/pbs` — the PBS-API client** (the agent's SECOND privileged external surface, + slice-1 discipline): TLS **fingerprint-pinned** to the PBS leaf cert (a spoofed PBS → + rejected, mirroring the PVE pin), **token auth** (`PBSAPIToken=:`; id from the + storage `username`, secret read at runtime from `/etc/pve/priv/storage/.pw` — referenced + by location, never logged/committed), typed, no shell. Methods: `Verify` (POST + `/admin/datastore//verify` → UPID), `Snapshots` (incl. the `verification` field), + `TaskStatus`/`WaitVerify` (node extracted from the UPID — `localhost` returns "unknown", the + spike B4 gotcha), `NodeFromUPID`. +- **The verify maintenance loop** (`pbs/verify.go`) — the cheap, key-free, ciphertext-level + integrity check (§8) on its OWN cadence (default 6h, the 5th daemon goroutine). It is a + reporting/maintenance task like the slice-5 watchdog: it does NOT go through the reconcile + gate/journal. Each cycle: trigger verify → poll task → re-list snapshots → record + per-snapshot `verify_state`. A failed verify is logged loudly. +- **`PBSSnapshot` reporting** — filled the stub (`namespace`/`backup_type`/`backup_id`/ + `backup_time`(RFC3339)/`size_bytes`/`owner`/`protected`/`encrypted` (from `files[].crypt-mode`) + /`verify_state` (ok|failed|**none** until verified)/`verify_upid`). New `PBSReporter` + collector seam + an in-memory `SnapshotStore`. Cross-repo golden (both repos, byte-identical) + + bidirectional key-set tests; hub `handler.go` parses `pbs_snapshots` and logs a **failed + verify `[WARN]`** (loudest offsite-DR signal). +- **Truthful backup mode** (`backup/runner.go`) — `Backup.mode` now reflects the ACTUAL vzdump + mode read from the task log (`backup mode: `), since PVE may downgrade snapshot→stop for a + stopped guest (spike B1); falls back to the requested mode if unparseable. +- **proxmox**: `Storage.Username` (parsed from the pbs storage config — the token id). +- **config** `BackupConfig.{PBSVerifyCadenceSeconds, PBSSecretDir}` (cadence 0→6h, <0 disabled). +- **`--selftest=pbs-verify`** — discover pbs storages → verify each → print the PBSSnapshot + records (covers the runbook's verify + list). Standalone on the host. + +### Notes +- Backup/restore-to-PBS reuse Phase A with no change (the restore-test runs with + `source_tier="pbs"` when fed a pbs volid). Zero-knowledge holds: verify is ciphertext-level, + the encryption key is never read here, and the PBS server has no client key (spike B6). +- Daemon runs cleanly with no pbs storage / verify disabled. `go test -race` covers the new + goroutine. Slice-3/4/5/6A surfaces, goldens, and adversarial tests intact. + ## v0.6.0-rc1 — slice 6 Phase A: backup + the self-restore-test (local target) (2026-06-09) Phase A of the backup/restore slice (doc 03 §8) — the agent's guest-level backup layer and diff --git a/CLAUDE.md b/CLAUDE.md index 794249f..b3202ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ - Module `gitea.dooplex.hu/admin/felhom-agent`; binary `felhom-agent` (`cmd/felhom-agent/`). - **Pure Go stdlib + `golang.org/x/crypto` only** — no web frameworks. - `go.mod` directive **go 1.25.0**; dep `golang.org/x/crypto v0.52.0` (declares go 1.25, will NOT build on Go 1.24). The **build server (192.168.0.180) runs go1.26.0** (upstream Go on PATH, backward-compatible). Build/run the agent there for live tests (same LAN as the demo host). -- Version: `version` var in `cmd/felhom-agent/main.go`, overridable via `-ldflags "-X main.version="`; `--version` flag. **Current: v0.6.0-rc1** (slice 6 Phase A: backup + the self-restore-test, local target; PBS = Phase B). Bump on meaningful changes + add a CHANGELOG entry. +- Version: `version` var in `cmd/felhom-agent/main.go`, overridable via `-ldflags "-X main.version="`; `--version` flag. **Current: v0.6.0** (slice 6 complete: backup + self-restore-test + the PBS offsite tier — verify + PBS-API client + PBSSnapshot reporting). Bump on meaningful changes + add a CHANGELOG entry. ## Layout @@ -29,6 +29,7 @@ internal/hub/ daemon: HostReport collector + Bearer client + resilient Lo internal/reconcile/ reconcile engine + reversibility gate + op journal + crash recovery + restore-test internal/storage/ storage-target observer + durable_id + fast-poll watchdog (slice 5) internal/backup/ vzdump backup runner + restore-test scheduler + report store (slice 6) +internal/pbs/ PBS-API client (fingerprint-pinned) + verify maintenance loop (slice 6 Phase B) ``` ## Proxmox model (the load-bearing rules) @@ -56,7 +57,8 @@ Built in slices, all on `main`: - **v0.5.0-rc1** — slice-5 **Phase A** (read-only, live): `internal/storage` — the `StorageTarget` wire contract (filled the slice-3 stub), `durable_id` derivation per type, the `Observer`, and the **storage watchdog** (third daemon goroutine; fast-poll → debounced out-of-band report on a known target's attach/disconnect). Hub ingest accepts/persists `storage_targets`; cross-repo golden byte-identical. - **v0.5.0** — slice-5 **Phase B** (the host-root surface): the `HostOps` seam + `SudoHostOps` (systemd `.mount` units by fs-UUID, detach, SMART, lvs) behind a **strict argument validator** (the adversarial matrix is the headline security test — hostile UUID/path/device refused with zero exec); SMART (SATA+NVMe) + thin-pool metadata enrichment; the watchdog's benign **re-mount response** (off the poll path); the **disk-grow executor** (`pct resize`, grow-only, benign) and **destructive storage ops** through the slice-4 gate (target-scoped; built + tested, inert live); `--selftest=storage [-watch]`; `configs/felhom-agent.sudoers`. - **v0.6.0-rc1** — slice-6 **Phase A** (backup + self-restore-test, local target): proxmox `DestroyLXC`/`Vzdump`-notes/`LatestBackupVolID`; `Engine.RunRestoreTest` (journaled scratch lifecycle: restore-to-new → net link-down → boot → verify running → defer teardown, all benign); `Recover` extended to reap a leaked scratch guest (Scratch journal flag, special-cased before the UPID path); `internal/backup` (runner + bulk-gap + cadence scheduler + report store); hub `Backup`/`RestoreTest` filled (cross-repo golden + hub logs a failed restore-test); `--selftest=backup`/`--selftest=restore-test`. Live-validated on demo-felhom. -- **Next: slice 6 Phase B (PBS)** — datastore on the USB, zero-knowledge key custody, restore-from-PBS, PBS integrity-verify (the lighter frequent check). Then slice 7 (provisioning + identity-reset + golden base, §9). +- **v0.6.0** — slice-6 **Phase B** (PBS offsite tier): `internal/pbs` — a fingerprint-pinned, token-authed PBS-API client (Verify/Snapshots/TaskStatus, node-from-UPID); the verify maintenance loop (own cadence, NOT gated/journaled — like the watchdog); `PBSSnapshot` reporting filled (cross-repo golden + hub failed-verify WARN); truthful vzdump mode from the task log; `--selftest=pbs-verify`. Backup/restore-to-PBS reuse Phase A unchanged. Live-validated against the spike's DooPlex PBS. +- **Next: slice 7 (provisioning + identity-reset + golden base, §9)** — the unified bring-up primitive; restore-overwrite + decommission executors the gate already guards; escrow + host-loss DR. ## Demo host (for live tests) diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index c47cbb8..8f58462 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -25,6 +25,7 @@ import ( "gitea.dooplex.hu/admin/felhom-agent/internal/config" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" applog "gitea.dooplex.hu/admin/felhom-agent/internal/log" + "gitea.dooplex.hu/admin/felhom-agent/internal/pbs" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" "gitea.dooplex.hu/admin/felhom-agent/internal/storage" @@ -32,7 +33,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.6.0-rc1" +var version = "0.6.0" func main() { var ( @@ -44,7 +45,7 @@ func main() { showVersion bool ) flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)") - flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup)") + flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records") flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup") flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only") flag.StringVar(&archive, "archive", "", "for --selftest=restore-test: the backup volid to restore (default: newest on the configured local target)") @@ -83,6 +84,8 @@ func main() { os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid)) case "restore-test": os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive)) + case "pbs-verify": + os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger)) } } @@ -189,7 +192,10 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { // latest restore-test result; the collector reads it via the BackupReporter / // RestoreTestReporter seams; the cadence scheduler writes it. backupStore := backup.NewStore() - collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, cfg.Hub.HostID, version, logger) + // PBS snapshot inventory + verify-state (slice 6 Phase B): the verify loop writes it; the + // collector reads it via the PBSReporter seam. + pbsStore := pbs.NewSnapshotStore() + collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, pbsStore, cfg.Hub.HostID, version, logger) loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger) interval := time.Duration(hcfg.PollSeconds) * time.Second @@ -276,19 +282,31 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { // OR the scratch band / restore storage is misconfigured — the daemon still runs. scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger) - // Run reconcile, the hub loop, the storage watchdog, and the restore-test scheduler - // concurrently; any one returning ends the daemon (ctx cancellation tears down the rest). - errc := make(chan error, 4) + // PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free, + // ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot + // verify-state. It is maintenance/reporting (NOT gated/journaled). Auto-discovers pbs + // storages from the PVE config each cycle; disabled cleanly (cadence<0) without crashing. + pbsLoop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{ + Targets: pbsTargetsFromPVE(cfg, px, logger), + Store: pbsStore, + Cadence: cfg.Backup.PBSVerifyCadence(), + Logger: logger, + }) + + // Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, and the + // PBS verify loop concurrently; any one returning ends the daemon (ctx cancel tears down rest). + errc := make(chan error, 5) go func() { errc <- engine.Run(ctx, interval) }() go func() { errc <- loop.Run(ctx) }() go func() { errc <- watchdog.Run(ctx) }() go func() { errc <- scheduler.Run(ctx) }() + go func() { errc <- pbsLoop.Run(ctx) }() err = <-errc - stop() // tear down the siblings on the first exit - <-errc // wait for the second - <-errc // wait for the third - <-errc // wait for the fourth + stop() // tear down the siblings on the first exit + for i := 0; i < 4; i++ { // wait for the other four + <-errc + } if err != nil && err != context.Canceled { logger.Error("daemon: exited with error", "err", err) return 1 @@ -296,6 +314,51 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { return 0 } +// pbsTargetsFromPVE returns a pbs.Targets closure that, each cycle, discovers the pbs +// storages from the PVE config and builds a fingerprint-pinned, token-authed client for each +// (token id from the storage `username`, secret read from /.pw). A storage +// whose secret/client can't be built is skipped with a warning (the loop still verifies the +// rest). The secret is read at runtime, never logged. +func pbsTargetsFromPVE(cfg config.Config, px *proxmox.Client, logger *slog.Logger) pbs.Targets { + return func(ctx context.Context) ([]pbs.Target, error) { + stores, err := px.ListStorage(ctx) + if err != nil { + return nil, err + } + var targets []pbs.Target + for _, s := range stores { + if s.Type != "pbs" { + continue + } + secret, err := readTrimmed(cfg.Backup.PBSSecretPath(s.Storage)) + if err != nil { + logger.Warn("pbs: cannot read token secret; skipping datastore", "storage", s.Storage, "err", err) + continue + } + c, err := pbs.NewClient(pbs.Config{Server: s.Server, Fingerprint: s.Fingerprint, TokenID: s.Username, Secret: secret}) + if err != nil { + logger.Warn("pbs: cannot build client; skipping datastore", "storage", s.Storage, "err", err) + continue + } + targets = append(targets, pbs.Target{Datastore: s.Datastore, Client: c}) + } + return targets, nil + } +} + +// readTrimmed reads a file and trims surrounding whitespace/newline (for the .pw secret). +func readTrimmed(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + s := strings.TrimSpace(string(b)) + if s == "" { + return "", fmt.Errorf("empty file %s", path) + } + return s, nil +} + // buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It // disables the cadence (returns a scheduler that just waits) when the cadence is off or the // scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the @@ -386,7 +449,7 @@ func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) return 1 } observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger) - collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, cfg.Hub.HostID, version, logger) + collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, nil, cfg.Hub.HostID, version, logger) ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() @@ -617,6 +680,56 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog return 0 } +// runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2 +// path), then lists + prints the resulting PBSSnapshot records (verify-state included). +// Standalone on the host. Covers the runbook's (c) verify and (d) list. +func runSelftestPBSVerify(ctx context.Context, cfg config.Config, logger *slog.Logger) int { + if err := cfg.Validate(); err != nil { + fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err) + return 1 + } + px, err := newProxmoxClient(cfg) + if err != nil { + fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err) + return 1 + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + + fmt.Printf("=== felhom-agent %s selftest=pbs-verify ===\n", version) + targets, err := pbsTargetsFromPVE(cfg, px, logger)(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, " [FAIL] discover pbs storages:", err) + return 1 + } + if len(targets) == 0 { + fmt.Println(" no pbs storages configured on this host") + return 0 + } + store := pbs.NewSnapshotStore() + loop := pbs.NewVerifyLoop(pbs.VerifyLoopOptions{ + Targets: func(context.Context) ([]pbs.Target, error) { return targets, nil }, + Store: store, + Logger: logger, + }) + // One synchronous verify+list pass over all datastores. + loop.RunOnce(ctx) + snaps := store.PBSSnapshots(ctx) + printJSON(fmt.Sprintf("%d pbs snapshot record(s)", len(snaps)), snaps) + failed := 0 + for _, s := range snaps { + if s.VerifyState == pbs.VerifyFailed { + failed++ + } + } + if failed > 0 { + fmt.Fprintf(os.Stderr, "=== selftest=pbs-verify: %d FAILED-verify snapshot(s) ===\n", failed) + return 1 + } + fmt.Printf("=== selftest=pbs-verify OK (%d snapshot(s) across %d datastore(s)) ===\n", len(snaps), len(targets)) + return 0 +} + // printJSON prints a labelled, indented JSON dump (best-effort) to stdout. func printJSON(label string, v any) { if b, err := json.MarshalIndent(v, " ", " "); err == nil { @@ -916,8 +1029,10 @@ func (f *selftestFlag) Set(v string) error { f.mode = "backup" case "restore-test": f.mode = "restore-test" + case "pbs-verify": + f.mode = "pbs-verify" default: - return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test)", v) + return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify)", v) } return nil } diff --git a/configs/agent.example.json b/configs/agent.example.json index 8d7a753..7bf44c9 100644 --- a/configs/agent.example.json +++ b/configs/agent.example.json @@ -41,7 +41,9 @@ "restore_storage": "local-lvm", "restore_test_cadence_seconds": 0, "scratch_vmid_min": 990000, - "scratch_vmid_max": 990009 + "scratch_vmid_max": 990009, + "pbs_verify_cadence_seconds": 0, + "pbs_secret_dir": "/etc/pve/priv/storage" }, "log_level": "info" } diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 9fe14e5..dc81100 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -25,6 +25,7 @@ type fakeBackupAPI struct { content []proxmox.StorageContent contentErr error vzdumps []proxmox.VzdumpOptions + logLines []string // returned by TaskLogTail (e.g. "INFO: backup mode: stop") } func (f *fakeBackupAPI) Vzdump(_ context.Context, o proxmox.VzdumpOptions) (string, error) { @@ -40,6 +41,9 @@ func (f *fakeBackupAPI) GuestConfig(_ context.Context, _ int) (proxmox.GuestConf func (f *fakeBackupAPI) StorageContent(_ context.Context, _ string) ([]proxmox.StorageContent, error) { return f.content, f.contentErr } +func (f *fakeBackupAPI) TaskLogTail(_ context.Context, _ string, _ int) ([]string, error) { + return f.logLines, nil +} // guestCfgWithMounts builds a GuestConfig whose Extra carries the given mpN strings. func guestCfgWithMounts(mps map[string]string) proxmox.GuestConfig { @@ -92,6 +96,23 @@ func TestBackup_SuccessResolvesArchiveAndBulkGap(t *testing.T) { } } +func TestBackup_ReportsActualModeFromTaskLog(t *testing.T) { + // Requested snapshot, but PVE used stop (stopped guest) — the report must reflect ACTUAL. + api := &fakeBackupAPI{ + vzdumpUPID: "UPID:vzdump:1", + content: []proxmox.StorageContent{{VolID: "v", Content: "backup", VMID: 9001, Size: 10, CTime: 1}}, + logLines: []string{"INFO: CT Name: spike", "INFO: backup mode: stop", "INFO: Finished"}, + } + r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", quiet()) + rec, err := r.Backup(context.Background(), 9001) + if err != nil { + t.Fatal(err) + } + if rec.Mode != "stop" { + t.Errorf("mode = %q, want the ACTUAL %q from the task log (not the requested snapshot)", rec.Mode, "stop") + } +} + func TestBackup_VzdumpFailureReturnsFailedRecord(t *testing.T) { api := &fakeBackupAPI{vzdumpErr: errors.New("vzdump boom")} r := NewBackupRunner(api, "local", "", "", quiet()) diff --git a/internal/backup/runner.go b/internal/backup/runner.go index d4feb06..161e716 100644 --- a/internal/backup/runner.go +++ b/internal/backup/runner.go @@ -19,6 +19,9 @@ type BackupAPI interface { WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error) StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error) + // TaskLogTail reads trailing task-log lines — used to read the ACTUAL vzdump mode + // (PVE may downgrade a requested snapshot to stop for a stopped guest — spike B1). + TaskLogTail(ctx context.Context, upid string, limit int) ([]string, error) } // BackupRunner orchestrates a crash-consistent vzdump to a local target and reports the @@ -85,6 +88,13 @@ func (r *BackupRunner) Backup(ctx context.Context, vmid int) (hub.Backup, error) rec.DurationSeconds = time.Since(start).Seconds() return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err) } + // Report the ACTUAL mode PVE used (it may downgrade snapshot→stop for a stopped + // guest — spike B1), read from the task log; fall back to the requested mode. + if lines, err := r.api.TaskLogTail(ctx, upid, 200); err == nil { + if actual := parseBackupMode(lines); actual != "" { + rec.Mode = actual + } + } } // Resolve the produced archive (volid + size) — the task status carries no result volid. @@ -139,6 +149,18 @@ func (r *BackupRunner) latestArchive(ctx context.Context, vmid int) (string, int return vol, size, nil } +// parseBackupMode extracts the actual mode from a vzdump task log line `… backup mode: ` +// (e.g. "INFO: backup mode: stop"). Returns "" if not found. +func parseBackupMode(lines []string) string { + const marker = "backup mode:" + for _, ln := range lines { + if i := strings.Index(ln, marker); i >= 0 { + return strings.TrimSpace(ln[i+len(marker):]) + } + } + return "" +} + // uncoveredMountpoints returns the mountpoint paths the guest vzdump EXCLUDES. LXC mount // points are OPT-IN to vzdump: a mpN with `backup=1` is covered; ANY other state — the // `backup=` token absent OR `backup=0` — is excluded. We deliberately treat unset as diff --git a/internal/config/config.go b/internal/config/config.go index 93b10fd..82a2e44 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,6 +47,13 @@ type BackupConfig struct { // always excluded. Defaults to 990000–990009. ScratchVMIDMin int `json:"scratch_vmid_min"` ScratchVMIDMax int `json:"scratch_vmid_max"` + + // PBS (slice 6 Phase B). The verify maintenance loop runs on its own cadence (cheaper + + // more frequent than the full restore-test); 0 → default (6h), negative → disabled. + PBSVerifyCadenceSeconds int `json:"pbs_verify_cadence_seconds"` + // PBSSecretDir holds the per-storage PBS token secret files (.pw). Default + // /etc/pve/priv/storage (PVE-managed, 0600). The agent reads it at runtime; never logged. + PBSSecretDir string `json:"pbs_secret_dir"` } // Default scratch VMID band + restore-test cadence. @@ -69,6 +76,28 @@ func (b BackupConfig) RestoreTestCadence() time.Duration { } } +// PBSVerifyCadence returns the verify-loop interval: positive as-is, 0 → 6h default, +// negative → 0 (disabled). +func (b BackupConfig) PBSVerifyCadence() time.Duration { + switch { + case b.PBSVerifyCadenceSeconds > 0: + return time.Duration(b.PBSVerifyCadenceSeconds) * time.Second + case b.PBSVerifyCadenceSeconds < 0: + return -1 // disabled (pbs.VerifyLoop treats <0 as disabled) + default: + return 6 * time.Hour + } +} + +// PBSSecretPath returns the path to a pbs storage's token-secret file. +func (b BackupConfig) PBSSecretPath(storageID string) string { + dir := b.PBSSecretDir + if dir == "" { + dir = "/etc/pve/priv/storage" + } + return dir + "/" + storageID + ".pw" +} + // ScratchBand returns the effective [min,max] scratch VMID band (defaults applied). func (b BackupConfig) ScratchBand() (min, max int) { min, max = b.ScratchVMIDMin, b.ScratchVMIDMax diff --git a/internal/hub/collect.go b/internal/hub/collect.go index d24313a..10eee71 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -42,6 +42,12 @@ type RestoreTestReporter interface { RestoreTests(ctx context.Context) []RestoreTest } +// PBSReporter is the slice-6-Phase-B seam the pbs verify loop plugs into (same pattern). +// Returns the agent's latest-known PBS snapshot inventory + verify-state. nil → empty. +type PBSReporter interface { + PBSSnapshots(ctx context.Context) []PBSSnapshot +} + // Collector builds a HostReport from read-only sources. All deps are behind narrow // interfaces for unit testing. type Collector struct { @@ -50,6 +56,7 @@ type Collector struct { storage StorageObserver backups BackupReporter restoreTests RestoreTestReporter + pbs PBSReporter hostID string agentVersion string logger *slog.Logger @@ -57,8 +64,8 @@ type Collector struct { } // NewCollector builds a collector. hostID echoes config.Hub.HostID; agentVersion is -// the binary version. storage/backups/restoreTests may be nil (their collections emit empty). -func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserver, backups BackupReporter, restoreTests RestoreTestReporter, hostID, agentVersion string, logger *slog.Logger) *Collector { +// the binary version. storage/backups/restoreTests/pbs may be nil (their collections emit empty). +func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserver, backups BackupReporter, restoreTests RestoreTestReporter, pbs PBSReporter, hostID, agentVersion string, logger *slog.Logger) *Collector { if logger == nil { logger = slog.Default() } @@ -68,6 +75,7 @@ func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserve storage: storage, backups: backups, restoreTests: restoreTests, + pbs: pbs, hostID: hostID, agentVersion: agentVersion, logger: logger, @@ -96,10 +104,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { StorageTargets: c.collectStorage(ctx), Backups: c.collectBackups(ctx), RestoreTests: c.collectRestoreTests(ctx), - PBSSnapshots: []PBSSnapshot{}, // Phase B + PBSSnapshots: c.collectPBSSnapshots(ctx), - AuditTail: []AuditEntry{}, - Cloudflared: Cloudflared{Status: c.cloudflaredStatus(ctx)}, + AuditTail: []AuditEntry{}, + Cloudflared: Cloudflared{Status: c.cloudflaredStatus(ctx)}, } return report, nil } @@ -198,6 +206,17 @@ func (c *Collector) collectRestoreTests(ctx context.Context) []RestoreTest { return []RestoreTest{} } +// collectPBSSnapshots reads the latest PBS snapshot inventory via the seam (nil → empty). +func (c *Collector) collectPBSSnapshots(ctx context.Context) []PBSSnapshot { + if c.pbs == nil { + return []PBSSnapshot{} + } + if s := c.pbs.PBSSnapshots(ctx); s != nil { + return s + } + return []PBSSnapshot{} +} + func (c *Collector) cloudflaredStatus(ctx context.Context) string { if c.cf == nil { return "unknown" diff --git a/internal/hub/collect_test.go b/internal/hub/collect_test.go index 2ce750b..9677dc5 100644 --- a/internal/hub/collect_test.go +++ b/internal/hub/collect_test.go @@ -33,7 +33,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) { obs := fakeObserver{targets: []StorageTarget{ {Name: "local-lvm", Type: StorageTypeLVMThin, State: StorageStateAttached, Reachable: true}, }} - c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, "h", "0.5.0", quietLogger()) + c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, nil, "h", "0.5.0", quietLogger()) r, err := c.Collect(context.Background()) if err != nil { t.Fatalf("Collect: %v", err) @@ -45,7 +45,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) { func TestCollect_StorageObserverErrorDegradesToEmpty(t *testing.T) { px := &fakePx{node: "n", ns: newTestNodeStatus()} - c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, nil, nil, "h", "0.5.0", quietLogger()) + c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, nil, nil, nil, "h", "0.5.0", quietLogger()) r, err := c.Collect(context.Background()) if err != nil { t.Fatalf("a storage observe error must not sink the heartbeat: %v", err) @@ -64,7 +64,7 @@ func TestCollect_HostAndGuests(t *testing.T) { }, cfg: map[int]proxmox.GuestConfig{100: {Cores: 2, Memory: 2048}}, } - c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "demo-host-01", "0.3.0", quietLogger()) + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "demo-host-01", "0.3.0", quietLogger()) r, err := c.Collect(context.Background()) if err != nil { t.Fatalf("Collect: %v", err) @@ -104,7 +104,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) { cfg: map[int]proxmox.GuestConfig{100: {Cores: 2}}, cfgErr: map[int]error{200: errors.New("config read failed")}, } - c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "h", "0.3.1", quietLogger()) + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.3.1", quietLogger()) r, err := c.Collect(context.Background()) if err != nil { t.Fatalf("a per-guest failure must NOT fail the whole report: %v", err) @@ -125,7 +125,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) { func TestCollect_NodeStatusFailureIsHardError(t *testing.T) { px := &fakePx{node: "n", nsErr: errors.New("proxmox down")} - c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "h", "0.3.0", quietLogger()) + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.3.0", quietLogger()) if _, err := c.Collect(context.Background()); err == nil { t.Fatal("NodeStatus failure must be a hard error (no useful report)") } @@ -133,7 +133,7 @@ func TestCollect_NodeStatusFailureIsHardError(t *testing.T) { func TestCollect_CloudflaredProbeErrorIsUnknown(t *testing.T) { px := &fakePx{node: "n", ns: newTestNodeStatus()} - c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, nil, nil, "h", "0.3.0", quietLogger()) + c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, nil, nil, nil, "h", "0.3.0", quietLogger()) r, err := c.Collect(context.Background()) if err != nil { t.Fatalf("cloudflared failure must not be fatal: %v", err) diff --git a/internal/hub/contract_test.go b/internal/hub/contract_test.go index d992895..108d8ea 100644 --- a/internal/hub/contract_test.go +++ b/internal/hub/contract_test.go @@ -59,8 +59,15 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) { Pass: true, Verified: "boot+running", TestedAt: "2026-06-09T11:05:00Z", DurationSeconds: 1, }, }, - PBSSnapshots: []PBSSnapshot{}, AuditTail: []AuditEntry{}, - Cloudflared: Cloudflared{Status: "active"}, + PBSSnapshots: []PBSSnapshot{ + { + Namespace: "root", BackupType: "ct", BackupID: "9001", BackupTime: "2026-06-09T14:18:33Z", + SizeBytes: 1, Owner: "felhom@pbs!n100", Protected: false, Encrypted: true, + VerifyState: "ok", VerifyUPID: "UPID:dooplex:x:verify:felhom-spike:u:", + }, + }, + AuditTail: []AuditEntry{}, + Cloudflared: Cloudflared{Status: "active"}, } b, _ := json.Marshal(report) var got map[string]any @@ -82,6 +89,8 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) { // slice-6 additions — backups[0] / restore_tests[0] key sets (the bidirectional guard). assertSameKeys(t, "backups[0]", firstElem(golden["backups"]), firstElem(got["backups"])) assertSameKeys(t, "restore_tests[0]", firstElem(golden["restore_tests"]), firstElem(got["restore_tests"])) + // slice-6-Phase-B addition — pbs_snapshots[0] key set. + assertSameKeys(t, "pbs_snapshots[0]", firstElem(golden["pbs_snapshots"]), firstElem(got["pbs_snapshots"])) } // field extracts a nested object value from a decoded JSON map (nil if absent/not a map). diff --git a/internal/hub/report.go b/internal/hub/report.go index 5fc037b..3526304 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -172,15 +172,15 @@ const ( // (the bulk-backup mechanism is slice 10). Cross-repo contract: keep byte-identical with // felhom.eu/hub and the bidirectional golden key-set test. type Backup struct { - TargetID string `json:"target_id"` // backup storage name (e.g. "local") - VMID int `json:"vmid"` // source guest - Archive string `json:"archive"` // produced vzdump volid - Mode string `json:"mode"` // snapshot | stop - CrashConsistent bool `json:"crash_consistent"` // always true this slice + TargetID string `json:"target_id"` // backup storage name (e.g. "local") + VMID int `json:"vmid"` // source guest + Archive string `json:"archive"` // produced vzdump volid + Mode string `json:"mode"` // snapshot | stop + CrashConsistent bool `json:"crash_consistent"` // always true this slice SizeBytes int64 `json:"size_bytes"` Success bool `json:"success"` Error string `json:"error,omitempty"` - StartedAt string `json:"started_at"` // RFC3339 + StartedAt string `json:"started_at"` // RFC3339 DurationSeconds float64 `json:"duration_seconds"` UncoveredVolumes []string `json:"uncovered_volumes"` // backup=0/unset mountpoints (bulk gap) } @@ -198,8 +198,26 @@ type RestoreTest struct { DurationSeconds float64 `json:"duration_seconds"` } -type PBSSnapshot struct{} // slice 6 Phase B: PBS snapshot inventory fields TBD -type AuditEntry struct{} // audit-log tail entry fields TBD +// PBSSnapshot is one PBS (offsite) snapshot's inventory + integrity state (doc 03 §8, slice +// 6 Phase B). Sourced from the PBS API (internal/pbs). `verify_state` is the load-bearing +// field — "none" until a verify runs, then "ok"/"failed" (a failed verify is the loudest +// offsite-DR signal). `encrypted` is derived from the snapshot's data crypt-mode +// (zero-knowledge: the PBS server can't read it). Cross-repo contract — byte-identical golden +// + bidirectional key-set test, the slice-5/6 pattern. +type PBSSnapshot struct { + Namespace string `json:"namespace"` // "root" = default ns + BackupType string `json:"backup_type"` // ct | vm + BackupID string `json:"backup_id"` + BackupTime string `json:"backup_time"` // RFC3339 + SizeBytes int64 `json:"size_bytes"` + Owner string `json:"owner"` + Protected bool `json:"protected"` + Encrypted bool `json:"encrypted"` + VerifyState string `json:"verify_state"` // ok | failed | none + VerifyUPID string `json:"verify_upid,omitempty"` +} + +type AuditEntry struct{} // audit-log tail entry fields TBD // ControlEnvelope is the hub's 200 response to a host-report. This slice the agent // adopts ONLY PollIntervalSeconds; the rest are reserved/forward-compat fields it diff --git a/internal/hub/testdata/host-report.golden.json b/internal/hub/testdata/host-report.golden.json index 8778f0a..5f66eb9 100644 --- a/internal/hub/testdata/host-report.golden.json +++ b/internal/hub/testdata/host-report.golden.json @@ -111,7 +111,20 @@ "duration_seconds": 38.2 } ], - "pbs_snapshots": [], + "pbs_snapshots": [ + { + "namespace": "root", + "backup_type": "ct", + "backup_id": "9001", + "backup_time": "2026-06-09T14:18:33Z", + "size_bytes": 2518889256, + "owner": "felhom@pbs!n100", + "protected": false, + "encrypted": true, + "verify_state": "ok", + "verify_upid": "UPID:dooplex:00034582:5269BDD7:00000005:6A282176:verify:felhom-spike:felhom@pbs!n100:" + } + ], "cloudflared": { "status": "active" }, "audit_tail": [] } diff --git a/internal/pbs/client.go b/internal/pbs/client.go new file mode 100644 index 0000000..0ec0057 --- /dev/null +++ b/internal/pbs/client.go @@ -0,0 +1,209 @@ +package pbs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Client is the PBS-API client for ONE PBS server. Construct with NewClient. It is pure (no +// logger — it must never log the token secret); callers log around it. +type Client struct { + base string // https://:/api2/json + authHeader string // "PBSAPIToken=:" — SECRET; never logged + http *http.Client +} + +// Config builds a Client. Secret is read by the caller from /etc/pve/priv/storage/.pw at +// runtime (referenced by location, never committed). +type Config struct { + Server string // PBS host (no scheme), e.g. "192.168.0.180" + Port int // default 8007 + Fingerprint string // SHA-256 of the PBS leaf cert (colons optional) + TokenID string // e.g. "felhom@pbs!n100" (from storage.cfg `username`) + Secret string // token secret (from .pw) + Timeout time.Duration +} + +// NewClient builds a fingerprint-pinned, token-authed PBS client. +func NewClient(cfg Config) (*Client, error) { + if cfg.Server == "" || cfg.Fingerprint == "" || cfg.TokenID == "" || cfg.Secret == "" { + return nil, fmt.Errorf("pbs: NewClient needs server, fingerprint, tokenid and secret") + } + tlsCfg, err := pinnedTLS(cfg.Fingerprint) + if err != nil { + return nil, err + } + port := cfg.Port + if port == 0 { + port = 8007 + } + timeout := cfg.Timeout + if timeout == 0 { + timeout = 30 * time.Second + } + return &Client{ + base: fmt.Sprintf("https://%s:%d/api2/json", cfg.Server, port), + authHeader: "PBSAPIToken=" + cfg.TokenID + ":" + cfg.Secret, + http: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + }, + }, nil +} + +// Verify triggers a datastore verify (POST /admin/datastore//verify) and returns the +// task UPID. With no snapshots it verifies the whole datastore; the cheap, key-free, +// ciphertext-level integrity check (doc 03 §8). Needs the token's Datastore.Verify (in +// DatastoreAdmin). Per-snapshot scoping is a future refinement; whole-datastore is the spike- +// proven path. +func (c *Client) Verify(ctx context.Context, datastore string, _ ...string) (string, error) { + var out struct { + Data string `json:"data"` + } + path := fmt.Sprintf("/admin/datastore/%s/verify", url.PathEscape(datastore)) + if err := c.do(ctx, http.MethodPost, path, &out); err != nil { + return "", err + } + return out.Data, nil +} + +// Snapshot is one PBS snapshot as the API returns it (GET /admin/datastore//snapshots). +type Snapshot struct { + BackupType string `json:"backup-type"` // ct | vm + BackupID string `json:"backup-id"` + BackupTime int64 `json:"backup-time"` // epoch seconds + Size int64 `json:"size"` + Owner string `json:"owner"` + Protected bool `json:"protected"` + Namespace string `json:"ns"` // "" = root namespace + Verification *struct { + State string `json:"state"` // ok | failed + UPID string `json:"upid"` + } `json:"verification"` + Files []struct { + Filename string `json:"filename"` + CryptMode string `json:"crypt-mode"` // encrypt | sign-only | (none) + Size int64 `json:"size"` + } `json:"files"` +} + +// Snapshots lists the datastore's snapshots (incl. the verification field). +func (c *Client) Snapshots(ctx context.Context, datastore string) ([]Snapshot, error) { + var out struct { + Data []Snapshot `json:"data"` + } + path := fmt.Sprintf("/admin/datastore/%s/snapshots", url.PathEscape(datastore)) + if err := c.do(ctx, http.MethodGet, path, &out); err != nil { + return nil, err + } + return out.Data, nil +} + +// TaskStatus is the subset of a PBS task status we need. +type TaskStatus struct { + Status string `json:"status"` // running | stopped + ExitStatus string `json:"exitstatus"` // present once stopped ("OK" or an error) + Node string `json:"node"` +} + +// Running reports whether the task is still executing. +func (t TaskStatus) Running() bool { return t.Status == "running" } + +// OK reports whether the task stopped successfully. +func (t TaskStatus) OK() bool { return t.Status == "stopped" && t.ExitStatus == "OK" } + +// TaskStatus polls a verify task. The PBS node name is extracted from the UPID — querying the +// wrong node (e.g. "localhost") returns exitstatus "unknown" (the spike B4 gotcha). +func (c *Client) TaskStatus(ctx context.Context, upid string) (TaskStatus, error) { + node := NodeFromUPID(upid) + if node == "" { + return TaskStatus{}, fmt.Errorf("pbs: cannot extract node from UPID %q", upid) + } + var out struct { + Data TaskStatus `json:"data"` + } + path := fmt.Sprintf("/nodes/%s/tasks/%s/status", url.PathEscape(node), url.PathEscape(upid)) + if err := c.do(ctx, http.MethodGet, path, &out); err != nil { + return TaskStatus{}, err + } + return out.Data, nil +} + +// WaitVerify polls a verify task until it stops or ctx/timeout elapses (best-effort — a poll +// failure returns the error; the caller re-lists snapshots for the authoritative state). +func (c *Client) WaitVerify(ctx context.Context, upid string, poll, timeout time.Duration) error { + if poll <= 0 { + poll = 2 * time.Second + } + if timeout <= 0 { + timeout = 10 * time.Minute + } + deadline := time.Now().Add(timeout) + t := time.NewTicker(poll) + defer t.Stop() + for { + st, err := c.TaskStatus(ctx, upid) + if err != nil { + return err + } + if !st.Running() { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("pbs: verify task %s did not finish within %s", upid, timeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } +} + +// NodeFromUPID extracts the node from a PBS/PVE UPID ("UPID:::..."). +func NodeFromUPID(upid string) string { + parts := strings.Split(upid, ":") + if len(parts) < 2 || parts[0] != "UPID" { + return "" + } + return parts[1] +} + +// do performs a request, sets the token auth header, and decodes the JSON body into out. The +// auth header carries the secret and is NEVER logged. +func (c *Client) do(ctx context.Context, method, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, method, c.base+path, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", c.authHeader) + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("pbs: %s %s: %w", method, path, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("pbs: %s %s -> HTTP %d: %s", method, path, resp.StatusCode, trimBody(body)) + } + if out != nil { + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("pbs: decoding %s %s: %w", method, path, err) + } + } + return nil +} + +func trimBody(b []byte) string { + s := strings.TrimSpace(string(b)) + if len(s) > 300 { + return s[:300] + "…" + } + return s +} diff --git a/internal/pbs/client_test.go b/internal/pbs/client_test.go new file mode 100644 index 0000000..1c52fe9 --- /dev/null +++ b/internal/pbs/client_test.go @@ -0,0 +1,161 @@ +package pbs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// fingerprintOf returns the SHA-256 hex of a TLS test server's leaf cert (what the client pins). +func fingerprintOf(ts *httptest.Server) string { + sum := sha256.Sum256(ts.Certificate().Raw) + return hex.EncodeToString(sum[:]) +} + +// newPBSTestServer spins up a TLS server routing requests to fn, plus its fingerprint. +func newPBSTestServer(t *testing.T, fn http.HandlerFunc) (*httptest.Server, string) { + t.Helper() + ts := httptest.NewTLSServer(fn) + t.Cleanup(ts.Close) + return ts, fingerprintOf(ts) +} + +// hostPort splits an httptest URL "https://127.0.0.1:PORT" into host + port for NewClient. +func hostPort(t *testing.T, url string) (string, int) { + t.Helper() + hp := strings.TrimPrefix(url, "https://") + host, port, ok := strings.Cut(hp, ":") + if !ok { + t.Fatalf("bad url %q", url) + } + p := 0 + for _, c := range port { + p = p*10 + int(c-'0') + } + return host, p +} + +// TestClient_FingerprintPinEnforced is the headline: a WRONG fingerprint is rejected; the +// RIGHT one succeeds (mirrors the slice-1 PVE pin test). +func TestClient_FingerprintPinEnforced(t *testing.T) { + ts, fp := newPBSTestServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(`{"data":[]}`)) + }) + host, port := hostPort(t, ts.URL) + + // Wrong fingerprint → connection rejected at the TLS pin. + wrong := strings.Repeat("ab", 32) + bad, _ := NewClient(Config{Server: host, Port: port, Fingerprint: wrong, TokenID: "u@pbs!t", Secret: "s"}) + if _, err := bad.Snapshots(context.Background(), "ds"); err == nil || !strings.Contains(err.Error(), "pin mismatch") { + t.Fatalf("wrong fingerprint must be rejected with a pin mismatch, got %v", err) + } + + // Correct fingerprint → succeeds. + good, err := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "u@pbs!t", Secret: "s"}) + if err != nil { + t.Fatal(err) + } + if _, err := good.Snapshots(context.Background(), "ds"); err != nil { + t.Fatalf("correct fingerprint should connect: %v", err) + } +} + +// TestClient_TokenHeaderAndNeverLogged asserts the auth header is the PBS token form and that +// the secret is not exposed by the client's printed form. +func TestClient_TokenHeaderAndNeverLogged(t *testing.T) { + const secret = "super-secret-token-value-xyz" + var gotAuth string + ts, fp := newPBSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.Write([]byte(`{"data":[]}`)) + }) + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "felhom@pbs!n100", Secret: secret}) + if _, err := c.Snapshots(context.Background(), "ds"); err != nil { + t.Fatal(err) + } + if gotAuth != "PBSAPIToken=felhom@pbs!n100:"+secret { + t.Errorf("auth header = %q, want PBSAPIToken=:", gotAuth) + } + // The secret lives only in the unexported authHeader and is never handed to a logger (the + // client takes no logger). The "never logged" property of the verify LOOP — which does log + // — is asserted in TestVerifyLoop_NeverLogsSecret (verify_test.go). +} + +func TestClient_VerifyParsesUPID(t *testing.T) { + ts, fp := newPBSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.Contains(r.URL.Path, "/admin/datastore/ds/verify") { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + } + w.Write([]byte(`{"data":"UPID:dooplex:00034582:5269BDD7:00000005:6A282176:verify:ds:felhom@pbs!n100:"}`)) + }) + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "u@pbs!t", Secret: "s"}) + upid, err := c.Verify(context.Background(), "ds") + if err != nil || !strings.HasPrefix(upid, "UPID:dooplex:") { + t.Fatalf("Verify upid=%q err=%v", upid, err) + } +} + +func TestClient_SnapshotsMapToHub(t *testing.T) { + body := `{"data":[ + {"backup-type":"ct","backup-id":"9001","backup-time":1781014713,"size":2518889256, + "owner":"felhom@pbs!n100","protected":false, + "verification":{"state":"ok","upid":"UPID:dooplex:..:verify:ds:u:"}, + "files":[{"filename":"pct.conf.blob","crypt-mode":"encrypt","size":240}, + {"filename":"index.json.blob","crypt-mode":"sign-only","size":646}]}, + {"backup-type":"ct","backup-id":"9002","backup-time":1781000000,"size":10,"owner":"x", + "files":[{"filename":"root.pxar.didx","crypt-mode":"encrypt","size":10}]} + ]}` + ts, fp := newPBSTestServer(t, func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(body)) }) + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "u@pbs!t", Secret: "s"}) + snaps, err := c.Snapshots(context.Background(), "ds") + if err != nil || len(snaps) != 2 { + t.Fatalf("snaps=%d err=%v", len(snaps), err) + } + + // First: verified ok, encrypted, RFC3339 time, default namespace. + h0 := snaps[0].ToHub() + if h0.VerifyState != VerifyOK || !h0.Encrypted || h0.Namespace != "root" { + t.Errorf("h0 = %+v, want verify=ok encrypted=true ns=root", h0) + } + if h0.BackupTime != "2026-06-09T14:18:33Z" { + t.Errorf("backup_time = %q, want RFC3339 UTC", h0.BackupTime) + } + // Second: NO verification field → verify_state "none". + if h1 := snaps[1].ToHub(); h1.VerifyState != VerifyNone { + t.Errorf("absent verification must map to %q, got %q", VerifyNone, h1.VerifyState) + } +} + +func TestNodeFromUPID(t *testing.T) { + cases := map[string]string{ + "UPID:dooplex:00034582:5269BDD7:00000005:6A282176:verify:ds:u:": "dooplex", + "UPID:demo-felhom:00:00:00:00:vzdump:9001:root@pam:": "demo-felhom", + "not-a-upid": "", + "": "", + } + for in, want := range cases { + if got := NodeFromUPID(in); got != want { + t.Errorf("NodeFromUPID(%q) = %q, want %q", in, got, want) + } + } +} + +func TestNormalizeFingerprint(t *testing.T) { + if _, err := normalizeFingerprint("3b:95:5a"); err == nil { + t.Error("short fingerprint must error") + } + if _, err := normalizeFingerprint(strings.Repeat("g", 64)); err == nil { + t.Error("non-hex must error") + } + got, err := normalizeFingerprint("3B:95:" + strings.Repeat("a", 60)) + if err != nil || strings.Contains(got, ":") || got != strings.ToLower(got) { + t.Errorf("normalize = %q err=%v (want lowercased, colons stripped)", got, err) + } +} diff --git a/internal/pbs/doc.go b/internal/pbs/doc.go new file mode 100644 index 0000000..2b35b0f --- /dev/null +++ b/internal/pbs/doc.go @@ -0,0 +1,23 @@ +// Package pbs is the agent's PBS (Proxmox Backup Server) API client + the verify +// maintenance loop (doc 03 §8, slice 6 Phase B). +// +// PBS is a SEPARATE server (its own host:8007, its own API + token auth), distinct from the +// PVE proxmox.Client — so this is the agent's SECOND privileged external surface and gets the +// same slice-1 discipline: TLS fingerprint-pinned to the PBS leaf cert, token auth, typed, +// context-aware, NO shell. +// +// The spike (felhom.eu/documentation/tests/phase5-pbs-spike-findings.md) proved that +// backup-to-PBS and restore-from-PBS reuse Phase A UNCHANGED (PBS is just a storage target + +// a volid). The only genuinely new code here is: +// - client.go — the PBS-API client: Verify, Snapshots, TaskStatus (node from the UPID). +// - report.go — Snapshot → hub.PBSSnapshot mapping + a SnapshotStore implementing the hub +// PBSReporter seam (the collector reads it; hub does not import pbs). +// - verify.go — the verify maintenance loop on its OWN cadence (the cheap, frequent, +// ciphertext-level integrity check — needs NO encryption key, unlike the +// full self-restore-test). It is a reporting/maintenance task like the +// slice-5 watchdog: it does NOT go through the reconcile gate/journal. +// +// The encryption key is never needed here (verify is ciphertext-level), and the token secret +// is read at runtime from /etc/pve/priv/storage/.pw — referenced by location, never +// logged or committed (zero-knowledge holds; the PBS server has no client key — spike B6). +package pbs diff --git a/internal/pbs/pin.go b/internal/pbs/pin.go new file mode 100644 index 0000000..370d764 --- /dev/null +++ b/internal/pbs/pin.go @@ -0,0 +1,48 @@ +package pbs + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "fmt" + "strings" +) + +// pinnedTLS builds a tls.Config that pins the PBS server's leaf cert by SHA-256 — the same +// model as the PVE client (proxmox/tls.go). PBS serves a self-signed cert, so we disable the +// default chain check but enforce an exact-cert match: a spoofed PBS presents a different +// fingerprint and is rejected. fingerprint is hex with optional colons (the form in +// /etc/pve/storage.cfg and the slice-5 durable_id). +func pinnedTLS(fingerprint string) (*tls.Config, error) { + want, err := normalizeFingerprint(fingerprint) + if err != nil { + return nil, err + } + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the exact-cert pin below + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return fmt.Errorf("pbs: TLS pin: peer presented no certificate") + } + got := sha256.Sum256(rawCerts[0]) + if hex.EncodeToString(got[:]) != want { + return fmt.Errorf("pbs: TLS pin mismatch: server cert sha256 does not match configured fingerprint") + } + return nil + }, + }, nil +} + +// normalizeFingerprint lowercases and strips colons/whitespace, validating a 64-char +// (32-byte) hex SHA-256. +func normalizeFingerprint(fp string) (string, error) { + s := strings.ToLower(strings.NewReplacer(":", "", " ", "", "\t", "").Replace(fp)) + if len(s) != 64 { + return "", fmt.Errorf("pbs: fingerprint must be a SHA-256 (64 hex chars), got %d", len(s)) + } + if _, err := hex.DecodeString(s); err != nil { + return "", fmt.Errorf("pbs: fingerprint is not valid hex: %w", err) + } + return s, nil +} diff --git a/internal/pbs/report.go b/internal/pbs/report.go new file mode 100644 index 0000000..dee998e --- /dev/null +++ b/internal/pbs/report.go @@ -0,0 +1,84 @@ +package pbs + +import ( + "context" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// verify-state constants for the reported PBSSnapshot. +const ( + VerifyOK = "ok" + VerifyFailed = "failed" + VerifyNone = "none" // no verify has run yet (PBS omits the verification field) +) + +// ToHub maps a PBS API Snapshot to the hub.PBSSnapshot wire record (slice 6 Phase B). The +// backup-time epoch becomes RFC3339; `encrypted` is derived from the data files' +// crypt-mode (any "encrypt" → encrypted); verify_state is "none" until a verify runs. +func (s Snapshot) ToHub() hub.PBSSnapshot { + ns := s.Namespace + if ns == "" { + ns = "root" + } + out := hub.PBSSnapshot{ + Namespace: ns, + BackupType: s.BackupType, + BackupID: s.BackupID, + BackupTime: time.Unix(s.BackupTime, 0).UTC().Format(time.RFC3339), + SizeBytes: s.Size, + Owner: s.Owner, + Protected: s.Protected, + Encrypted: s.encrypted(), + VerifyState: VerifyNone, + } + if s.Verification != nil { + out.VerifyState = s.Verification.State + out.VerifyUPID = s.Verification.UPID + } + return out +} + +// encrypted reports whether the snapshot's DATA is client-side encrypted (any data file with +// crypt-mode "encrypt"; index.json is "sign-only" and is ignored). +func (s Snapshot) encrypted() bool { + for _, f := range s.Files { + if f.CryptMode == "encrypt" { + return true + } + } + return false +} + +// SnapshotStore holds the latest reported PBS snapshots per datastore — the point-in-time +// state the host-report surfaces. The verify loop writes it; the collector reads it via the +// hub PBSReporter seam. Mutex-guarded (concurrent collector vs loop). +type SnapshotStore struct { + mu sync.Mutex + byDatastore map[string][]hub.PBSSnapshot +} + +// NewSnapshotStore builds an empty store. +func NewSnapshotStore() *SnapshotStore { + return &SnapshotStore{byDatastore: map[string][]hub.PBSSnapshot{}} +} + +// Record replaces the snapshot set for a datastore. +func (s *SnapshotStore) Record(datastore string, snaps []hub.PBSSnapshot) { + s.mu.Lock() + defer s.mu.Unlock() + s.byDatastore[datastore] = snaps +} + +// PBSSnapshots implements hub.PBSReporter — all known snapshots across datastores. +func (s *SnapshotStore) PBSSnapshots(context.Context) []hub.PBSSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + out := []hub.PBSSnapshot{} + for _, snaps := range s.byDatastore { + out = append(out, snaps...) + } + return out +} diff --git a/internal/pbs/verify.go b/internal/pbs/verify.go new file mode 100644 index 0000000..c21aa31 --- /dev/null +++ b/internal/pbs/verify.go @@ -0,0 +1,127 @@ +package pbs + +import ( + "context" + "log/slog" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// DefaultVerifyCadence is the verify maintenance interval — more frequent than the full +// self-restore-test, because it's the cheap, key-free, ciphertext-level integrity check (§8). +const DefaultVerifyCadence = 6 * time.Hour + +// Target is one PBS datastore to verify, with its client. +type Target struct { + Datastore string + Client *Client +} + +// Targets resolves the current set of PBS datastores to verify (re-derived each cycle from +// the PVE storage config — wired in main.go so pbs stays decoupled from how clients are built). +type Targets func(ctx context.Context) ([]Target, error) + +// VerifyLoop is the verify maintenance loop (slice 6 Phase B). It runs on its OWN cadence and +// is a reporting/maintenance task like the slice-5 watchdog — it does NOT go through the +// reconcile gate/journal (it mutates no guest). Each cycle, per datastore: trigger a verify → +// poll the task → re-list snapshots → record the per-snapshot verify-state for the report. +type VerifyLoop struct { + targets Targets + store *SnapshotStore + cadence time.Duration + logger *slog.Logger +} + +// VerifyLoopOptions configures a VerifyLoop. +type VerifyLoopOptions struct { + Targets Targets + Store *SnapshotStore + Cadence time.Duration // 0 → default 6h; negative → disabled + Logger *slog.Logger +} + +// NewVerifyLoop builds a VerifyLoop. +func NewVerifyLoop(opts VerifyLoopOptions) *VerifyLoop { + logger := opts.Logger + if logger == nil { + logger = slog.Default() + } + cadence := opts.Cadence + if cadence == 0 { + cadence = DefaultVerifyCadence + } + return &VerifyLoop{targets: opts.Targets, store: opts.Store, cadence: cadence, logger: logger} +} + +// Run verifies on the cadence until ctx is cancelled. It does an immediate first pass (so a +// freshly-started agent reports snapshot inventory + verify-state promptly), then on each +// tick. A negative cadence (or nil targets/store) disables it. Returns nil on cancellation. +func (l *VerifyLoop) Run(ctx context.Context) error { + if l.cadence < 0 || l.targets == nil || l.store == nil { + l.logger.Info("pbs: verify loop disabled") + <-ctx.Done() + return nil + } + l.logger.Info("pbs: verify loop starting", "cadence", l.cadence) + l.tick(ctx) // immediate inventory + verify + t := time.NewTicker(l.cadence) + defer t.Stop() + for { + select { + case <-ctx.Done(): + l.logger.Info("pbs: verify loop shutting down", "reason", ctx.Err()) + return nil + case <-t.C: + l.tick(ctx) + } + } +} + +// RunOnce performs a single synchronous verify+list pass over all targets (used by the +// selftest harness and the live runbook). Same work as one cadence tick. +func (l *VerifyLoop) RunOnce(ctx context.Context) { l.tick(ctx) } + +// tick verifies + re-lists each target datastore once. Deterministic enough to drive directly +// in tests. A per-target error is logged and skipped (other datastores still report). +func (l *VerifyLoop) tick(ctx context.Context) { + targets, err := l.targets(ctx) + if err != nil { + l.logger.Warn("pbs: verify loop could not resolve targets; skipping", "err", err) + return + } + for _, t := range targets { + l.verifyOne(ctx, t) + } +} + +// verifyOne triggers a verify, waits for it, then re-lists + records the snapshots' state. +func (l *VerifyLoop) verifyOne(ctx context.Context, t Target) { + if upid, err := t.Client.Verify(ctx, t.Datastore); err != nil { + // Verify-trigger failure is non-fatal: still re-list so we report current state. + l.logger.Warn("pbs: verify trigger failed; reporting current snapshot state", "datastore", t.Datastore, "err", err) + } else if err := t.Client.WaitVerify(ctx, upid, 2*time.Second, 30*time.Minute); err != nil { + l.logger.Warn("pbs: verify task wait failed; reporting current snapshot state", "datastore", t.Datastore, "err", err) + } + + snaps, err := t.Client.Snapshots(ctx, t.Datastore) + if err != nil { + l.logger.Warn("pbs: snapshot list failed", "datastore", t.Datastore, "err", err) + return + } + out := make([]hub.PBSSnapshot, 0, len(snaps)) + failed := 0 + for _, s := range snaps { + h := s.ToHub() + if h.VerifyState == VerifyFailed { + failed++ + } + out = append(out, h) + } + l.store.Record(t.Datastore, out) + if failed > 0 { + l.logger.Error("pbs: datastore has FAILED-verify snapshots", "datastore", t.Datastore, "failed", failed, "total", len(out)) + } else { + l.logger.Info("pbs: verify cycle complete", "datastore", t.Datastore, "snapshots", len(out)) + } +} diff --git a/internal/pbs/verify_test.go b/internal/pbs/verify_test.go new file mode 100644 index 0000000..0f75e97 --- /dev/null +++ b/internal/pbs/verify_test.go @@ -0,0 +1,100 @@ +package pbs + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// verifyServer is a fake PBS that answers verify POSTs, task status, and a snapshot list whose +// verification.state is controlled by `state` ("" → no verification field → none). +func verifyServer(t *testing.T, state string) (*httptest.Server, string) { + t.Helper() + verification := "" + if state != "" { + verification = `"verification":{"state":"` + state + `","upid":"UPID:dooplex:0:0:0:0:verify:ds:u:"},` + } + body := `{"data":[{"backup-type":"ct","backup-id":"9001","backup-time":1781014713,"size":10,"owner":"u",` + + verification + `"files":[{"filename":"root.pxar.didx","crypt-mode":"encrypt","size":10}]}]}` + return newPBSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/verify"): + w.Write([]byte(`{"data":"UPID:dooplex:0:0:0:0:verify:ds:u:"}`)) + case strings.Contains(r.URL.Path, "/status"): + w.Write([]byte(`{"data":{"status":"stopped","exitstatus":"OK","node":"dooplex"}}`)) + default: // snapshots + w.Write([]byte(body)) + } + }) +} + +func TestVerifyLoop_RecordsState(t *testing.T) { + ts, fp := verifyServer(t, "ok") + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "u@pbs!t", Secret: "s"}) + store := NewSnapshotStore() + loop := NewVerifyLoop(VerifyLoopOptions{ + Targets: func(context.Context) ([]Target, error) { return []Target{{Datastore: "ds", Client: c}}, nil }, + Store: store, + }) + loop.RunOnce(context.Background()) + + snaps := store.PBSSnapshots(context.Background()) + if len(snaps) != 1 || snaps[0].VerifyState != VerifyOK { + t.Fatalf("loop should record 1 ok snapshot, got %+v", snaps) + } +} + +func TestVerifyLoop_FailedVerifyRecorded(t *testing.T) { + ts, fp := verifyServer(t, "failed") + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "u@pbs!t", Secret: "s"}) + store := NewSnapshotStore() + NewVerifyLoop(VerifyLoopOptions{ + Targets: func(context.Context) ([]Target, error) { return []Target{{Datastore: "ds", Client: c}}, nil }, + Store: store, + }).RunOnce(context.Background()) + if s := store.PBSSnapshots(context.Background()); len(s) != 1 || s[0].VerifyState != VerifyFailed { + t.Fatalf("failed verify must be recorded, got %+v", s) + } +} + +// TestVerifyLoop_NeverLogsSecret runs a full cycle with the loop's logger capturing output and +// asserts the token secret never appears in any log line. +func TestVerifyLoop_NeverLogsSecret(t *testing.T) { + const secret = "tok-secret-DO-NOT-LOG-7f3a" + ts, fp := verifyServer(t, "ok") + host, port := hostPort(t, ts.URL) + c, _ := NewClient(Config{Server: host, Port: port, Fingerprint: fp, TokenID: "felhom@pbs!n100", Secret: secret}) + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + loop := NewVerifyLoop(VerifyLoopOptions{ + Targets: func(context.Context) ([]Target, error) { return []Target{{Datastore: "ds", Client: c}}, nil }, + Store: NewSnapshotStore(), + Logger: logger, + }) + loop.RunOnce(context.Background()) + if strings.Contains(buf.String(), secret) { + t.Fatalf("the verify loop logged the token secret") + } +} + +func TestVerifyLoop_DisabledByNegativeCadence(t *testing.T) { + loop := NewVerifyLoop(VerifyLoopOptions{ + Targets: func(context.Context) ([]Target, error) { return nil, nil }, + Store: NewSnapshotStore(), + Cadence: -1, + }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + cancel() + if err := <-done; err != nil { + t.Fatalf("disabled loop Run should return nil on cancel, got %v", err) + } +} diff --git a/internal/proxmox/types.go b/internal/proxmox/types.go index 626b1d6..833b2ea 100644 --- a/internal/proxmox/types.go +++ b/internal/proxmox/types.go @@ -163,6 +163,7 @@ type Storage struct { Share string `json:"share,omitempty"` // cifs share name Datastore string `json:"datastore,omitempty"` // pbs datastore name Fingerprint string `json:"fingerprint,omitempty"` // pbs server cert fingerprint + Username string `json:"username,omitempty"` // pbs auth id, e.g. "felhom@pbs!n100" VGName string `json:"vgname,omitempty"` // lvm/lvmthin volume group ThinPool string `json:"thinpool,omitempty"` // lvmthin pool LV name }