slice 7 Phase 1: unified bring-up reconcile job (provision + guest-loss DR) (v0.8.0)

The shared front half of provision and guest-loss DR as a journaled reconcile job (internal/reconcile/bringup.go), mirroring the restore-test's crash-safety but keeping the guest on success and applying a scenario-specific identity policy. Agent-only; no hub/wire change. Grounded by the slice-7 bring-up spike (commit 3342993): F1/F3/F4. - RunBringUp: restore -> reset identity -> size -> attach mounts -> start link-up; verdict is liveness (waitRunning), success KEEPS the guest. - identity policy: provision = fresh MAC (net0 sans hwaddr -> PVE regen) + hostname, host-side; machine-id/host-keys regenerate guest-side (systemd + baked golden unit). dr_guest_loss = preserve continuity (keep hostname; keep MAC unless KeepMAC=false). - compensating rollback: mid-flight failure destroys the just-created guest (SameTxnCreated provenance, gated); new Rollback journal flag + Recover.recoverBringUp reap a half-built guest from a crash. - F4: coalesced config PUT + bounded retry on the transient PVE config-lock 500 only. - --selftest=bring-up (mode/archive/vmid/hostname/keep). - configs/build-golden.sh: validated golden recipe incl. the F3 first-boot host-key unit. - doc-03 §9 + identity-reset settled/implemented. Deferred (stated): provisioning back half -> slice 8; host-loss DR + escrow consumption and the BringUpSpec source (hub desired-state) -> slice 10. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 21:27:49 +02:00
parent 9f6753de0f
commit 57405c1a99
10 changed files with 1184 additions and 73 deletions
@@ -33,7 +33,7 @@ import (

 // version is the agent version. Overridable at build time with
 // -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
-var version = "0.7.0"
+var version = "0.8.0"

 func main() {
 	var (
@@ -42,13 +42,19 @@ func main() {
 		vmid        int
 		watch       time.Duration
 		archive     string
+		mode        string
+		hostname    string
+		keep        bool
 		showVersion bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records")
-	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; tears down unless -keep)")
+	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
 	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
-	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test: the backup volid to restore (default: newest on the configured local target)")
+	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
+	flag.StringVar(&mode, "mode", "provision", "for --selftest=bring-up: `provision` (golden, fresh identity) | `dr` (customer backup, preserve continuity)")
+	flag.StringVar(&hostname, "hostname", "", "for --selftest=bring-up provision: the hostname to set on the new guest")
+	flag.BoolVar(&keep, "keep", false, "for --selftest=bring-up: KEEP the guest instead of tearing it down at the end")
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.Parse()

@@ -86,6 +92,8 @@ func main() {
 		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
 	case "pbs-verify":
 		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
+	case "bring-up":
+		os.Exit(runSelftestBringUp(context.Background(), cfg, logger, mode, archive, vmid, hostname, keep))
 	}
 }

@@ -695,6 +703,103 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
 	return 0
 }

+// runSelftestBringUp runs the REAL journaled bring-up job (slice 7) on-demand: restore →
+// reset identity → size → attach mounts → start link-up, then tears the guest down by default
+// (a selftest must not leave a guest running) unless -keep. -mode picks provision (golden, fresh
+// identity) or dr (customer backup, preserve continuity). It first Recovers, so a leaked guest
+// from a prior crashed bring-up is reaped before this run.
+func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Logger, mode, archive string, vmid int, hostname string, keep bool) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	var bmode reconcile.BringUpMode
+	switch mode {
+	case "provision":
+		bmode = reconcile.ModeProvision
+	case "dr":
+		bmode = reconcile.ModeDRGuestLoss
+	default:
+		fmt.Fprintf(os.Stderr, "selftest=bring-up: -mode must be provision|dr (got %q)\n", mode)
+		return 2
+	}
+	if archive == "" || vmid <= 0 {
+		fmt.Fprintln(os.Stderr, "selftest=bring-up requires -archive <volid> and -vmid <N>")
+		return 2
+	}
+	if cfg.Backup.RestoreStorage == "" {
+		fmt.Fprintln(os.Stderr, "selftest=bring-up requires backup.restore_storage in config")
+		return 2
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	queue := reconcile.NewQueue()
+	defer queue.Close()
+	var journal *reconcile.Journal
+	if jp := reconcileJournalPath(cfg); jp != "" {
+		if err := os.MkdirAll(filepath.Dir(jp), 0o700); err == nil {
+			if j, err := reconcile.OpenJournal(jp); err == nil {
+				journal = j
+				defer journal.Close()
+			}
+		}
+	}
+	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
+	engine := reconcile.NewEngine(reconcile.EngineOptions{
+		API: px, Queue: queue, Journal: journal, Gate: gate, HostID: cfg.Hub.HostID, Logger: logger,
+	})
+
+	fmt.Printf("=== felhom-agent %s selftest=bring-up (mode=%s vmid=%d) ===\n", version, mode, vmid)
+	fmt.Println("  --- recover: reaping any half-built guest from a prior crashed bring-up ---")
+	rec := engine.Recover(ctx)
+	fmt.Printf("  recover: examined=%d bring_up_rolled_back=%d bring_up_clean=%d scratch_destroyed=%d\n",
+		rec.Examined, rec.BringUpRolledBack, rec.BringUpClean, rec.ScratchDestroyed)
+
+	spec := reconcile.BringUpSpec{
+		Mode: bmode, Archive: archive, VMID: vmid, RestoreStorage: cfg.Backup.RestoreStorage,
+		Hostname: hostname, KeepMAC: bmode == reconcile.ModeDRGuestLoss,
+	}
+	fmt.Printf("  bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
+	res := engine.RunBringUp(ctx, spec)
+	printJSON("bring-up record", res)
+
+	if res.Err != nil || !res.Pass {
+		fmt.Fprintf(os.Stderr, "  [FAIL] bring-up (vmid %d): %v\n", vmid, res.Err)
+		// On failure the job already compensating-rolled-back; nothing to tear down.
+		return 1
+	}
+	fmt.Printf("  [OK] vmid %d up (boot+running) in %s; MAC=%s\n", res.VMID, res.Duration.Round(time.Second), res.AssignedMAC)
+	if len(res.StartWarnings) > 0 {
+		fmt.Printf("  start warnings (recognized=%v): %v\n", res.WarningsRecognized, res.StartWarnings)
+	}
+
+	if keep {
+		fmt.Printf("=== selftest=bring-up OK — guest %d KEPT (-keep) ===\n", vmid)
+		return 0
+	}
+	// A selftest must not leave a guest running. Tear it down (out-of-band, like the spike).
+	fmt.Printf("  --- teardown: destroying selftest guest %d ---\n", vmid)
+	upid, err := px.DestroyLXC(ctx, vmid)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy: %v (destroy vmid %d manually)\n", err, vmid)
+		return 1
+	}
+	if upid != "" {
+		if _, err := px.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
+			fmt.Fprintf(os.Stderr, "  [WARN] teardown destroy task: %v (check vmid %d)\n", err, vmid)
+			return 1
+		}
+	}
+	fmt.Printf("=== selftest=bring-up OK (vmid %d brought up, verified, torn down) ===\n", vmid)
+	return 0
+}
+
 // runSelftestPBSVerify discovers the pbs storages, triggers a verify on each (the new §2
 // path), then lists + prints the resulting PBSSnapshot records (verify-state included).
 // Standalone on the host. Covers the runbook's (c) verify and (d) list.
@@ -1046,8 +1151,10 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "restore-test"
 	case "pbs-verify":
 		f.mode = "pbs-verify"
+	case "bring-up":
+		f.mode = "bring-up"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up)", v)
 	}
 	return nil
 }