feat(reconcile): re-assert pool membership after restore-over-existing (campaign-2 R2, v0.74.0)

Pool membership is what lets the pool-scoped token reach a guest; pct restore
--pool sets it only at CREATE, so a restore over an existing VMID drops the guest
from the felhom pool and 403s the next restore-test/DR on VM.Audit. This empty-pool
state is the true root cause of the campaign's "R1" (bind-mount restore failing was
a symptom — restore-test's existing bind neutralization never ran without config-read).

Add Client.PoolAddVMID (PUT /pools, additive+idempotent, Pool.Allocate) and call it
in bring-up after liveness when spec.Pool!="" — warn-not-fail on a hiccup (liveness
wins). B3 scratch-teardown 403 diagnosed as a cascade (restoretest already passes
Pool). Role/ACL untouched. Tests + red-proof.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

internal/proxmox/mutate.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 )
 
 // Async mutating operations. Each is API-token-covered (the FelhomAgent role) and
@@ -185,6 +186,36 @@ func (c *Client) SetConfig(ctx context.Context, vmid int, params map[string]stri
 	return c.dataString(ctx, http.MethodPut, path, v)
 }
 
+// PoolAddVMID adds a guest to a PVE pool via PUT /pools/{pool} with vms={vmid}.
+//
+// Membership is what makes a pool-scoped token (FelhomAgentGuest @ /pool/<pool>) reach a guest: the
+// grant applies only to pool MEMBERS. `pct restore --pool` sets membership at CREATE, but a restore
+// OVER AN EXISTING VMID (the P9/host-loss finale) does NOT re-apply it — so a destroy-restore
+// silently drops the guest from the pool and 403s the NEXT restore-test/DR (campaign-2 R2). This
+// re-asserts it after such a restore.
+//
+// PVE semantics: `PUT /pools/{poolid}` with `vms` is ADDITIVE (a merge) — `delete=1` is required to
+// REMOVE, so passing a single vmid adds it without disturbing existing members. Adding a guest that
+// is already a member is treated as a no-op success (idempotent): PVE reports "already" in the error
+// body, which we swallow. Requires Pool.Allocate at /pool/<pool> (the token has it).
+func (c *Client) PoolAddVMID(ctx context.Context, pool string, vmid int) error {
+	if pool == "" || vmid == 0 {
+		return fmt.Errorf("proxmox: PoolAddVMID needs pool and vmid")
+	}
+	v := url.Values{}
+	v.Set("vms", strconv.Itoa(vmid))
+	path := "/pools/" + url.PathEscape(pool)
+	_, err := c.dataString(ctx, http.MethodPut, path, v)
+	if err != nil {
+		// Idempotent: a guest already in the pool is success, not a failure.
+		if ae, ok := err.(*APIError); ok && strings.Contains(strings.ToLower(ae.Body), "already") {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 // ResizeLXC grows a guest volume via PUT /nodes/{node}/lxc/{vmid}/resize
 // (token-covered: VM.Config.Disk + Datastore.AllocateSpace). Returns the UPID.
 //

internal/proxmox/pool_test.go

@@ -0,0 +1,96 @@
+package proxmox
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"testing"
+)
+
+// PoolAddVMID issues PUT /pools/{pool} with vms={vmid} and treats an already-member error as success.
+func TestPoolAddVMID_PUTShape(t *testing.T) {
+	var gotMethod, gotPath, gotVMS string
+	d := &mockDoer{fn: func(r *http.Request) (*http.Response, error) {
+		gotMethod = r.Method
+		gotPath = r.URL.Path
+		body, _ := io.ReadAll(r.Body) // the client encodes url.Values into the request body
+		for _, kv := range splitAmp(string(body)) {
+			if k, v, ok := cut(kv, "="); ok && k == "vms" {
+				gotVMS = v
+			}
+		}
+		return jsonResp(http.StatusOK, `{"data":null}`), nil
+	}}
+	c := newTestClient(d)
+	if err := c.PoolAddVMID(context.Background(), "felhom", 9201); err != nil {
+		t.Fatalf("PoolAddVMID: %v", err)
+	}
+	if gotMethod != http.MethodPut {
+		t.Errorf("method = %q, want PUT", gotMethod)
+	}
+	if gotPath != "/api2/json/pools/felhom" {
+		t.Errorf("path = %q, want /api2/json/pools/felhom", gotPath)
+	}
+	if gotVMS != "9201" {
+		t.Errorf("vms param = %q, want 9201", gotVMS)
+	}
+}
+
+// Idempotent: an "already in pool" error from PVE is swallowed as success.
+func TestPoolAddVMID_IdempotentOnAlreadyMember(t *testing.T) {
+	d := &mockDoer{fn: func(_ *http.Request) (*http.Response, error) {
+		return jsonResp(http.StatusInternalServerError, `{"data":null,"errors":{"vms":"VM 9201 is already in pool 'felhom'"}}`), nil
+	}}
+	c := newTestClient(d)
+	if err := c.PoolAddVMID(context.Background(), "felhom", 9201); err != nil {
+		t.Fatalf("already-member should be idempotent success, got: %v", err)
+	}
+}
+
+// A real failure (e.g. a 403 with no "already") is surfaced, not swallowed.
+func TestPoolAddVMID_RealErrorSurfaces(t *testing.T) {
+	d := &mockDoer{fn: func(_ *http.Request) (*http.Response, error) {
+		return jsonResp(http.StatusForbidden, `Permission check failed (/pool/felhom, Pool.Allocate)`), nil
+	}}
+	c := newTestClient(d)
+	if err := c.PoolAddVMID(context.Background(), "felhom", 9201); err == nil {
+		t.Fatal("expected a real 403 to surface, got nil")
+	}
+}
+
+func TestPoolAddVMID_Validation(t *testing.T) {
+	c := newTestClient(&mockDoer{fn: func(_ *http.Request) (*http.Response, error) {
+		t.Fatal("should not issue a request on bad input")
+		return nil, nil
+	}})
+	if err := c.PoolAddVMID(context.Background(), "", 9201); err == nil {
+		t.Error("empty pool should error")
+	}
+	if err := c.PoolAddVMID(context.Background(), "felhom", 0); err == nil {
+		t.Error("zero vmid should error")
+	}
+}
+
+// tiny helpers (avoid pulling strings.Split into an assertion path that could mask a bug)
+func splitAmp(s string) []string {
+	var out []string
+	cur := ""
+	for _, r := range s {
+		if r == '&' {
+			out = append(out, cur)
+			cur = ""
+			continue
+		}
+		cur += string(r)
+	}
+	return append(out, cur)
+}
+
+func cut(s, sep string) (string, string, bool) {
+	for i := 0; i+len(sep) <= len(s); i++ {
+		if s[i:i+len(sep)] == sep {
+			return s[:i], s[i+len(sep):], true
+		}
+	}
+	return s, "", false
+}