v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)

The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→
teardown) that closes "a backup you haven't restored isn't a backup". All benign
(reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local
target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent.

- proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID.
- reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down
  pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a
  leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent).
- internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence
  scheduler (4th daemon goroutine, default 24h) + in-memory report store.
- hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical +
  bidirectional key-set tests; hub handler logs a FAILED restore-test prominently.
- config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

cmd/felhom-agent/main.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/config"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
 	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
@@ -31,7 +32,7 @@ import (
 
 // version is the agent version. Overridable at build time with
 // -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
-var version = "0.5.1"
+var version = "0.6.0-rc1"
 
 func main() {
 	var (
@@ -39,12 +40,14 @@ func main() {
 		selftest    selftestFlag
 		vmid        int
 		watch       time.Duration
+		archive     string
 		showVersion bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report to the hub; `storage` = observe storage targets (+ -watch for the live watchdog)")
-	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task (the reversible snapshot/rollback exercise)")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup)")
+	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup")
 	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
+	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test: the backup volid to restore (default: newest on the configured local target)")
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.Parse()
 
@@ -76,6 +79,10 @@ func main() {
 		os.Exit(runSelftestHub(context.Background(), cfg, logger))
 	case "storage":
 		os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
+	case "backup":
+		os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
+	case "restore-test":
+		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
 	}
 }
 
@@ -178,7 +185,11 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	hostReader := storage.NewProcHostReader()
 	hostOps := newHostOps(cfg, logger)
 	observer := storage.NewObserver(px, hostReader, hostOps, logger)
-	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, cfg.Hub.HostID, version, logger)
+	// Backup + restore-test state store (slice 6): holds the latest backup-per-target +
+	// latest restore-test result; the collector reads it via the BackupReporter /
+	// RestoreTestReporter seams; the cadence scheduler writes it.
+	backupStore := backup.NewStore()
+	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, backupStore, backupStore, cfg.Hub.HostID, version, logger)
 	loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
 	interval := time.Duration(hcfg.PollSeconds) * time.Second
 
@@ -256,20 +267,28 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	})
 
 	// Crash recovery (doc 03 §10): resolve any op that was in flight when the agent
-	// last died BEFORE issuing new mutations. With an empty journal this is a no-op.
+	// last died BEFORE issuing new mutations. For slice 6 this is load-bearing — a
+	// restore-test scratch guest leaked by a mid-test crash is torn down here.
 	engine.Recover(ctx)
 
-	// Run reconcile, the hub loop, and the storage watchdog concurrently; any one
-	// returning ends the daemon (then ctx cancellation tears the others down).
-	errc := make(chan error, 3)
+	// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
+	// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
+	// OR the scratch band / restore storage is misconfigured — the daemon still runs.
+	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)
+
+	// Run reconcile, the hub loop, the storage watchdog, and the restore-test scheduler
+	// concurrently; any one returning ends the daemon (ctx cancellation tears down the rest).
+	errc := make(chan error, 4)
 	go func() { errc <- engine.Run(ctx, interval) }()
 	go func() { errc <- loop.Run(ctx) }()
 	go func() { errc <- watchdog.Run(ctx) }()
+	go func() { errc <- scheduler.Run(ctx) }()
 
 	err = <-errc
 	stop() // tear down the siblings on the first exit
 	<-errc // wait for the second
 	<-errc // wait for the third
+	<-errc // wait for the fourth
 	if err != nil && err != context.Canceled {
 		logger.Error("daemon: exited with error", "err", err)
 		return 1
@@ -277,6 +296,35 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	return 0
 }
 
+// buildRestoreTestScheduler constructs the restore-test cadence scheduler from config. It
+// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
+// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
+// machinery still works on-demand via --selftest=restore-test.
+func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, logger *slog.Logger) *backup.Scheduler {
+	cadence := cfg.Backup.RestoreTestCadence()
+	if cadence > 0 {
+		if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
+			logger.Warn("daemon: restore-test cadence disabled (config invalid)", "err", err)
+			cadence = 0
+		}
+	}
+	min, max := cfg.Backup.ScratchBand()
+	runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom restore-test", logger)
+	return backup.NewScheduler(backup.SchedulerOptions{
+		Runner: engine,
+		Pick:   runner.PickRestoreCandidate,
+		Store:  store,
+		Spec: reconcile.RestoreTestSpec{
+			RestoreStorage: cfg.Backup.RestoreStorage,
+			ScratchMin:     min,
+			ScratchMax:     max,
+			SourceTier:     "local",
+		},
+		Cadence: cadence,
+		Logger:  logger,
+	})
+}
+
 // reconcileJournalPath chooses the op-journal path: a `journal.log` sibling of the
 // configured nonce store (both are durable agent state), falling back to the standard
 // host state dir when the nonce store is unset.
@@ -338,7 +386,7 @@ func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger)
 		return 1
 	}
 	observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
-	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, cfg.Hub.HostID, version, logger)
+	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, nil, nil, cfg.Hub.HostID, version, logger)
 
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
@@ -460,6 +508,122 @@ func smartCounters(s hub.SmartSummary) string {
 	return " " + strings.Join(parts, " ")
 }
 
+// runSelftestBackup runs one vzdump of -vmid to the configured local backup target and
+// prints the resulting Backup record. Standalone (no hub). Live + benign.
+func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	if vmid == 0 {
+		fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
+		return 2
+	}
+	if cfg.Backup.LocalBackupTarget == "" {
+		fmt.Fprintln(os.Stderr, "selftest=backup requires backup.local_backup_target in config (a content=backup storage)")
+		return 2
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+
+	fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, cfg.Backup.LocalBackupTarget)
+	runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom selftest", logger)
+	rec, err := runner.Backup(ctx, vmid)
+	printJSON("backup record", rec)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "  [FAIL] backup:", err)
+		return 1
+	}
+	fmt.Printf("=== selftest=backup OK (crash-consistent=%v, archive=%s) ===\n", rec.CrashConsistent, rec.Archive)
+	return 0
+}
+
+// runSelftestRestoreTest runs one self-restore-test (restore → net-link-down → boot → verify
+// running → teardown) of -archive (or the newest backup on the local target) into a scratch
+// guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
+// crashed test is reaped before this run.
+func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest=restore-test:", err)
+		return 2
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// Build a journaled engine so a leaked scratch from a prior crash is recoverable.
+	queue := reconcile.NewQueue()
+	defer queue.Close()
+	var journal *reconcile.Journal
+	if jp := reconcileJournalPath(cfg); jp != "" {
+		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
+			if j, err := reconcile.OpenJournal(jp); err == nil {
+				journal = j
+				defer journal.Close()
+			}
+		}
+	}
+	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
+	engine := reconcile.NewEngine(reconcile.EngineOptions{
+		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
+	})
+
+	fmt.Printf("=== felhom-agent %s selftest=restore-test ===\n", version)
+	fmt.Println("  --- recover: reaping any leaked scratch from a prior crashed test ---")
+	rec := engine.Recover(ctx)
+	fmt.Printf("  recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)
+
+	if archive == "" {
+		runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "", logger)
+		archive, err = runner.PickRestoreCandidate(ctx)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "  [FAIL] pick backup:", err)
+			return 1
+		}
+		if archive == "" {
+			fmt.Fprintln(os.Stderr, "  [FAIL] no backup available on", cfg.Backup.LocalBackupTarget, "(run --selftest=backup first)")
+			return 1
+		}
+	}
+	min, max := cfg.Backup.ScratchBand()
+	fmt.Printf("  restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
+	res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
+		Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
+		ScratchMin: min, ScratchMax: max, SourceTier: "local",
+	})
+	printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
+	if res.Skipped {
+		fmt.Println("=== selftest=restore-test SKIPPED (no free scratch VMID in band) ===")
+		return 0
+	}
+	if res.Err != nil || !res.Pass {
+		fmt.Fprintf(os.Stderr, "  [FAIL] restore-test (scratch %d): %v\n", res.ScratchVMID, res.Err)
+		return 1
+	}
+	fmt.Printf("=== selftest=restore-test OK (scratch %d restored+booted+verified+torn-down in %s) ===\n", res.ScratchVMID, res.Duration.Round(time.Second))
+	return 0
+}
+
+// printJSON prints a labelled, indented JSON dump (best-effort) to stdout.
+func printJSON(label string, v any) {
+	if b, err := json.MarshalIndent(v, "  ", "  "); err == nil {
+		fmt.Printf("  --- %s ---\n  %s\n", label, string(b))
+	}
+}
+
 // runSelftestRead loads config, builds the API client, and runs the read-only
 // queries against the live host, printing a short health report. It mutates
 // nothing. Missing/invalid config is reported cleanly (no panic).
@@ -748,8 +912,12 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "hub"
 	case "storage":
 		f.mode = "storage"
+	case "backup":
+		f.mode = "backup"
+	case "restore-test":
+		f.mode = "restore-test"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test)", v)
 	}
 	return nil
 }