feat(reconcile): re-assert pool membership after restore-over-existing (campaign-2 R2, v0.74.0)

Pool membership is what lets the pool-scoped token reach a guest; pct restore --pool sets it only at CREATE, so a restore over an existing VMID drops the guest from the felhom pool and 403s the next restore-test/DR on VM.Audit. This empty-pool state is the true root cause of the campaign's "R1" (bind-mount restore failing was a symptom — restore-test's existing bind neutralization never ran without config-read). Add Client.PoolAddVMID (PUT /pools, additive+idempotent, Pool.Allocate) and call it in bring-up after liveness when spec.Pool!="" — warn-not-fail on a hiccup (liveness wins). B3 scratch-teardown 403 diagnosed as a cascade (restoretest already passes Pool). Role/ACL untouched. Tests + red-proof. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-07 18:42:19 +02:00
parent e04b75e1f8
commit ca0b169a4e
8 changed files with 278 additions and 0 deletions
@@ -344,6 +344,22 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
 		return
 	}

+	// 6b. Re-assert pool membership (campaign-2 R2). `pct restore --pool` sets membership only at
+	//     CREATE; a restore OVER AN EXISTING VMID (host-loss finale) does not re-apply it, silently
+	//     dropping the guest from the pool and 403-ing the NEXT restore-test/DR. Idempotent on a
+	//     fresh-VMID restore that already got membership. This runs AFTER liveness is proven: a
+	//     pool-add hiccup is surfaced LOUD as a warning but must NOT flip a healthy, running guest's
+	//     verdict to fail (membership matters for the next op, not this guest's boot).
+	if spec.Pool != "" {
+		if err := e.api.PoolAddVMID(ctx, spec.Pool, spec.VMID); err != nil {
+			e.logger.Error("bring-up: pool membership re-assert FAILED (next restore-test/DR may 403); guest is healthy",
+				"vmid", spec.VMID, "pool", spec.Pool, "err", err)
+			res.StartWarnings = append(res.StartWarnings, fmt.Sprintf("pool re-assert failed (pool=%s): %v", spec.Pool, err))
+		} else {
+			e.logger.Info("bring-up: pool membership re-asserted", "vmid", spec.VMID, "pool", spec.Pool)
+		}
+	}
+
 	// 7. Success — KEEP the guest; mark the owning entry terminal so Recover ignores it.
 	res.Pass = true
 	res.Verified = "boot+running"
@@ -629,3 +629,91 @@ func lockBackoffFast(t *testing.T) {
 	configLockBackoff = time.Millisecond
 	t.Cleanup(func() { configLockBackoff = prev })
 }
+
+// --- Part B (campaign-2 R2): pool re-assertion after bring-up ------------------------------------
+
+// poolAddsFor returns the recorded PoolAddVMID calls for a vmid.
+func poolAddsFor(api *fakeAPI, vmid int) []poolAddCall {
+	var out []poolAddCall
+	for _, c := range api.poolAdds {
+		if c.vmid == vmid {
+			out = append(out, c)
+		}
+	}
+	return out
+}
+
+// A bring-up with a Pool set MUST re-assert pool membership after the restore (so a
+// restore-over-existing that dropped membership is healed for the NEXT restore-test/DR).
+// COMPANION RED-PROOF: on the pre-fix code (no PoolAddVMID call) poolAddsFor is empty → this FAILS.
+func TestRunBringUp_ReassertsPoolMembership(t *testing.T) {
+	const vmid = 8100
+	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunBringUp(context.Background(), BringUpSpec{
+		Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
+		RestoreStorage: "local-lvm", Pool: "felhom",
+	})
+	if res.Err != nil || !res.Pass {
+		t.Fatalf("bring-up must pass, got %+v", res)
+	}
+	adds := poolAddsFor(api, vmid)
+	if len(adds) != 1 || adds[0].pool != "felhom" {
+		t.Fatalf("pool membership not re-asserted: poolAdds=%+v", api.poolAdds)
+	}
+}
+
+// No Pool → no PoolAddVMID call (a broad-token restore needs no pool).
+func TestRunBringUp_NoPool_NoPoolAdd(t *testing.T) {
+	const vmid = 8101
+	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunBringUp(context.Background(), BringUpSpec{
+		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
+		RestoreStorage: "local-lvm", Hostname: "h",
+	})
+	if res.Err != nil || !res.Pass {
+		t.Fatalf("bring-up must pass, got %+v", res)
+	}
+	if len(api.poolAdds) != 0 {
+		t.Fatalf("no Pool set → PoolAddVMID must not be called: %+v", api.poolAdds)
+	}
+}
+
+// A pool-add FAILURE must NOT flip a healthy, running guest's verdict — it surfaces as a LOUD warning
+// (membership matters for the NEXT op, not this guest's boot). Liveness wins.
+func TestRunBringUp_PoolAddFailure_WarnsButPasses(t *testing.T) {
+	const vmid = 8102
+	api := &fakeAPI{
+		cfg:        map[int]proxmox.GuestConfig{vmid: scratchCfg()},
+		poolAddErr: errors.New("proxmox: PUT /pools/felhom -> HTTP 500: transient"),
+	}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunBringUp(context.Background(), BringUpSpec{
+		Mode: ModeDRGuestLoss, Archive: "local:backup/cust.tar.zst", VMID: vmid,
+		RestoreStorage: "local-lvm", Pool: "felhom",
+	})
+	if res.Err != nil || !res.Pass {
+		t.Fatalf("a pool-add hiccup must NOT fail a healthy guest, got %+v", res)
+	}
+	// the guest is KEPT (not torn down) despite the pool-add failure.
+	if len(api.destroys) != 0 {
+		t.Fatalf("pool-add failure must not trigger teardown: %+v", api.destroys)
+	}
+	// the failure is surfaced as a warning.
+	found := false
+	for _, w := range res.StartWarnings {
+		if strings.Contains(w, "pool re-assert failed") {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("pool-add failure must surface as a warning: %+v", res.StartWarnings)
+	}
+}
@@ -49,6 +49,23 @@ type fakeAPI struct {
 	waits    []string
 	waitOpts []proxmox.WaitOptions // parallel to waits: the options each WaitTask was called with
 	listErr  error
+
+	// poolAdds records (pool, vmid) for each PoolAddVMID; poolAddErr backs the failure path.
+	poolAdds   []poolAddCall
+	poolAddErr error
+}
+
+type poolAddCall struct {
+	pool string
+	vmid int
+}
+
+func (f *fakeAPI) PoolAddVMID(_ context.Context, pool string, vmid int) error {
+	f.mu.Lock()
+	f.poolAdds = append(f.poolAdds, poolAddCall{pool: pool, vmid: vmid})
+	err := f.poolAddErr
+	f.mu.Unlock()
+	return err
 }

 type resizeCall struct {
@@ -172,6 +172,9 @@ type GuestAPI interface {
 	ResizeLXC(ctx context.Context, vmid int, disk, size string) (string, error)
 	// RestoreLXC restores an archive into a (fresh) vmid — the create path (slice 6). Async → UPID.
 	RestoreLXC(ctx context.Context, opts proxmox.RestoreLXCOptions) (string, error)
+	// PoolAddVMID re-asserts pool membership after a restore-over-existing (campaign-2 R2). Sync (no
+	// UPID); idempotent. Membership is what lets the pool-scoped token reach the guest next time.
+	PoolAddVMID(ctx context.Context, pool string, vmid int) error
 	// DestroyLXC destroys a guest — the scratch-teardown primitive (slice 6). Async → UPID.
 	// Destructive-class; the engine only ever issues it for an agent-tagged scratch guest
 	// (benign by provenance) via the gate.