0fabc15896
CORRECTION: I earlier reported that the restore-test would boot a scratch guest with the live guest's MAC/static island IP/hostname and break the control plane. That was WRONG — RunRestoreTest step 2 link-downs EVERY interface (withLinkDown, unit-tested) before the guest is ever started. The design already handled it. The real, narrower hazard: a restore that fails BEFORE step 2 (what the v0.100.0 wait bug caused) leaves a scratch holding the SOURCE guest's config verbatim, including onboot:1. If teardown also fails (403 missing VM.Allocate — PVE associates the pool only at restore completion), a host reboot would start that leaked clone alongside the original with NICs up. - proxmox.RestoreLXCOptions.ConfigOverrides: guest-config params applied AT RESTORE TIME. - The restore-test passes onboot=0 — at restore time, not after, because 'after' is exactly the path that leaks. NOT changed: the link-down step (already correct, the primary defence); the agent's Proxmox privileges (widening VM.Allocate to /vms would remove the accidental guard that stopped a destructive mid-restore teardown). restore_test_cadence_seconds was set to -1 on demo-felhom under the mistaken reading; re-enabled. Red-proof observed; full suite green (29 packages).
262 lines
12 KiB
Go
262 lines
12 KiB
Go
package proxmox
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Async mutating operations. Each is API-token-covered (the FelhomAgent role) and
|
|
// returns a UPID string; the caller MUST WaitTask on it and assert exitstatus OK.
|
|
// The HTTP 200 here is not proof of success (phase1-2 §1.3).
|
|
|
|
// BackupMode is the vzdump mode.
|
|
type BackupMode string
|
|
|
|
const (
|
|
// ModeStop: orderly guest shutdown -> backup -> restart. Highest consistency.
|
|
// For LXC the shutdown/restart is internal to vzdump and needs only VM.Backup
|
|
// (NOT VM.PowerMgmt) — phase1-2 §1.4.
|
|
ModeStop BackupMode = "stop"
|
|
// ModeSnapshot: lowest downtime; for an LXC this is crash-consistent only (no
|
|
// fsfreeze) — app-consistency is the controller's job (proxmox-platform.md §4.2).
|
|
ModeSnapshot BackupMode = "snapshot"
|
|
)
|
|
|
|
// RestoreLXCOptions parameterizes a restore. This is the PRIMARY create path:
|
|
// a token-authorized restore preserves features=nesting=1,keyctl=1 from the
|
|
// archive, so it needs no root (phase3 §B3). Fresh `pct create` with keyctl is
|
|
// the only root-fenced create (see Privileged.CreateGoldenLXC).
|
|
type RestoreLXCOptions struct {
|
|
VMID int // target VMID (fresh id)
|
|
Archive string // source archive volid, e.g. "local:backup/vzdump-lxc-9001-...tar.zst"
|
|
Storage string // target storage for the rootfs, e.g. "local-lvm"
|
|
Force bool // overwrite an existing VMID (destructive — caller must have authority)
|
|
// Pool allocates the restored guest INTO a PVE pool (pct restore --pool). "" = no pool. Under a
|
|
// pool-scoped token this is REQUIRED for the restore to authorize (VM.Allocate + Pool.Allocate are
|
|
// granted at /pool/<pool>, not /), and it makes the guest reachable by the scoped token afterwards
|
|
// (SPIKE-pool-scoped-acl-2026-07-01). Empty is valid — a broad-token restore needs no pool.
|
|
Pool string
|
|
// MountOverrides overrides specific mountpoints at restore time (mpN -> full value, e.g.
|
|
// "local-lvm:1,mp=/data,backup=0"). A restore param takes precedence over the archive's own
|
|
// mpN. The restore-test uses this to neutralize a SOURCE host bind-mount mountpoint (slice-10
|
|
// data drive) — vzrestore refuses to restore a bind mount under the privsep token ("restoring
|
|
// 'mpN' to bind mount is only possible for root"); replacing it with a throwaway volume needs
|
|
// no root and the boot-verify doesn't need the drive's data.
|
|
MountOverrides map[string]string
|
|
// ConfigOverrides sets arbitrary guest-config params AT RESTORE TIME (they take precedence over
|
|
// the archive's own values), for settings that must hold from the instant the guest exists —
|
|
// before any post-restore SetConfig could run.
|
|
//
|
|
// The restore-test uses it for `onboot=0`. A restore that fails BEFORE the post-restore config
|
|
// step leaves a scratch guest carrying the SOURCE guest's config verbatim, including
|
|
// `onboot: 1` — so a leaked scratch would auto-start on the next host reboot, with the source's
|
|
// MAC, static island IP and hostname. Observed live 2026-07-26. The normal path link-downs every
|
|
// NIC before boot, so this is defence in depth for the ABNORMAL path, where the leak happens.
|
|
ConfigOverrides map[string]string
|
|
}
|
|
|
|
// RestoreLXC restores an LXC from a vzdump/PBS archive via POST /nodes/{node}/lxc
|
|
// (restore=1). Returns the UPID. NOTE: pct restore preserves the source MAC +
|
|
// hostname — reset network identity before starting alongside the original
|
|
// (phase1-2 §2.2). Identity reset is a SetConfig call the caller makes after.
|
|
func (c *Client) RestoreLXC(ctx context.Context, opts RestoreLXCOptions) (string, error) {
|
|
if opts.VMID == 0 || opts.Archive == "" || opts.Storage == "" {
|
|
return "", fmt.Errorf("proxmox: RestoreLXC needs vmid, archive and storage")
|
|
}
|
|
v := url.Values{}
|
|
v.Set("vmid", strconv.Itoa(opts.VMID))
|
|
v.Set("ostemplate", opts.Archive) // pct restore source
|
|
v.Set("restore", "1")
|
|
v.Set("storage", opts.Storage)
|
|
if opts.Force {
|
|
v.Set("force", "1")
|
|
}
|
|
if opts.Pool != "" {
|
|
v.Set("pool", opts.Pool) // allocate into the pool (pool-scoped token needs this — see RestoreLXCOptions.Pool)
|
|
}
|
|
for k, val := range opts.MountOverrides {
|
|
v.Set(k, val) // e.g. mp0 -> "local-lvm:1,mp=/data,backup=0" (overrides the archive's mp0)
|
|
}
|
|
for k, val := range opts.ConfigOverrides {
|
|
v.Set(k, val) // e.g. onboot -> "0" (a leaked scratch must never auto-start)
|
|
}
|
|
return c.dataString(ctx, http.MethodPost, "/nodes/"+c.node+"/lxc", v)
|
|
}
|
|
|
|
// VzdumpOptions parameterizes a backup.
|
|
type VzdumpOptions struct {
|
|
VMID int
|
|
Storage string // a storage whose content includes "backup" (e.g. "local") — NOT local-lvm
|
|
Mode BackupMode // ModeStop | ModeSnapshot
|
|
Compress string // "zstd" (default), "lzo", "gzip", or "" for none
|
|
// Notes is the PVE `notes-template` for the backup (a template string PVE expands,
|
|
// e.g. with {{guestname}}/{{node}}). Optional.
|
|
Notes string
|
|
// PruneBackups is the PVE `--prune-backups` retention spec applied AFTER this backup, e.g.
|
|
// "keep-last=3". PVE prunes only THIS vmid's archives on THIS storage (the vzdump is vmid+storage
|
|
// scoped), so it never touches other guests/storages or PBS. Empty → no prune (legacy behaviour).
|
|
PruneBackups string
|
|
}
|
|
|
|
// Vzdump starts a backup via POST /nodes/{node}/vzdump. Returns the UPID. An
|
|
// agent-initiated vzdump is crash-consistent only for an LXC (no fsfreeze) —
|
|
// app-consistency needs the controller to quiesce first (slice 8).
|
|
func (c *Client) Vzdump(ctx context.Context, opts VzdumpOptions) (string, error) {
|
|
if opts.VMID == 0 || opts.Storage == "" || opts.Mode == "" {
|
|
return "", fmt.Errorf("proxmox: Vzdump needs vmid, storage and mode")
|
|
}
|
|
v := url.Values{}
|
|
v.Set("vmid", strconv.Itoa(opts.VMID))
|
|
v.Set("storage", opts.Storage)
|
|
v.Set("mode", string(opts.Mode))
|
|
if opts.Compress == "" {
|
|
opts.Compress = "zstd"
|
|
}
|
|
v.Set("compress", opts.Compress)
|
|
if opts.Notes != "" {
|
|
v.Set("notes-template", opts.Notes) // PVE 9.x param name (verified on demo)
|
|
}
|
|
if opts.PruneBackups != "" {
|
|
// Per-run retention: PVE prunes older archives of THIS vmid on THIS storage after the backup.
|
|
v.Set("prune-backups", opts.PruneBackups)
|
|
}
|
|
return c.dataString(ctx, http.MethodPost, "/nodes/"+c.node+"/vzdump", v)
|
|
}
|
|
|
|
// DestroyLXC destroys a guest via DELETE /nodes/{node}/lxc/{vmid}. Returns the UPID.
|
|
// `purge=1` also drops the guest from jobs/HA; `destroy-unreferenced-disks=1` reaps any
|
|
// orphaned volumes. This is the scratch-guest teardown primitive (slice 6); it is
|
|
// destructive-class and the caller MUST route it through the reversibility gate
|
|
// (benign only by agent-internal scratch/same-txn provenance — see reconcile.Classify).
|
|
func (c *Client) DestroyLXC(ctx context.Context, vmid int) (string, error) {
|
|
if vmid == 0 {
|
|
return "", fmt.Errorf("proxmox: DestroyLXC needs a vmid")
|
|
}
|
|
// DELETE takes NO request body on PVE (a form body → HTTP 501 "Unexpected content for
|
|
// method 'DELETE'"); the flags go in the query string. `force=1` destroys even a running
|
|
// guest (the scratch may still be booted at teardown), `purge=1` drops it from jobs/HA,
|
|
// `destroy-unreferenced-disks=1` reaps orphaned volumes.
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d?purge=1&destroy-unreferenced-disks=1&force=1", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodDelete, path, nil)
|
|
}
|
|
|
|
// Snapshot creates an LXC snapshot via POST /nodes/{node}/lxc/{vmid}/snapshot.
|
|
// A running, unprivileged LXC can be snapshotted on LVM-thin with no stop
|
|
// (phase1-2 §1.6) — this is the snapshot-before-change primitive.
|
|
func (c *Client) Snapshot(ctx context.Context, vmid int, snapname, description string) (string, error) {
|
|
if vmid == 0 || snapname == "" {
|
|
return "", fmt.Errorf("proxmox: Snapshot needs vmid and snapname")
|
|
}
|
|
v := url.Values{}
|
|
v.Set("snapname", snapname)
|
|
if description != "" {
|
|
v.Set("description", description)
|
|
}
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/snapshot", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodPost, path, v)
|
|
}
|
|
|
|
// Rollback rolls an LXC back to a snapshot via
|
|
// POST /nodes/{node}/lxc/{vmid}/snapshot/{snap}/rollback.
|
|
func (c *Client) Rollback(ctx context.Context, vmid int, snapname string) (string, error) {
|
|
if vmid == 0 || snapname == "" {
|
|
return "", fmt.Errorf("proxmox: Rollback needs vmid and snapname")
|
|
}
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/snapshot/%s/rollback", c.node, vmid, url.PathEscape(snapname))
|
|
return c.dataString(ctx, http.MethodPost, path, url.Values{})
|
|
}
|
|
|
|
// DeleteSnapshot removes an LXC snapshot via
|
|
// DELETE /nodes/{node}/lxc/{vmid}/snapshot/{snap}.
|
|
func (c *Client) DeleteSnapshot(ctx context.Context, vmid int, snapname string) (string, error) {
|
|
if vmid == 0 || snapname == "" {
|
|
return "", fmt.Errorf("proxmox: DeleteSnapshot needs vmid and snapname")
|
|
}
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/snapshot/%s", c.node, vmid, url.PathEscape(snapname))
|
|
return c.dataString(ctx, http.MethodDelete, path, nil)
|
|
}
|
|
|
|
// SetConfig applies config changes via PUT /nodes/{node}/lxc/{vmid}/config
|
|
// (e.g. memory, cores, net0, mpN with a backup flag). PVE may apply this
|
|
// synchronously (no UPID) — the returned string is empty in that case, and "" is
|
|
// not an error. When a UPID is returned, WaitTask on it.
|
|
//
|
|
// Identity reset after a restore (phase1-2 §2.2) is a SetConfig with
|
|
// params{"net0": "name=eth0,bridge=vmbr0,ip=dhcp"} (regenerates the MAC).
|
|
func (c *Client) SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error) {
|
|
if vmid == 0 || len(params) == 0 {
|
|
return "", fmt.Errorf("proxmox: SetConfig needs vmid and at least one param")
|
|
}
|
|
v := url.Values{}
|
|
for k, val := range params {
|
|
v.Set(k, val)
|
|
}
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/config", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodPut, path, v)
|
|
}
|
|
|
|
// PoolAddVMID adds a guest to a PVE pool via PUT /pools/{pool} with vms={vmid}.
|
|
//
|
|
// Membership is what makes a pool-scoped token (FelhomAgentGuest @ /pool/<pool>) reach a guest: the
|
|
// grant applies only to pool MEMBERS. `pct restore --pool` sets membership at CREATE, but a restore
|
|
// OVER AN EXISTING VMID (the P9/host-loss finale) does NOT re-apply it — so a destroy-restore
|
|
// silently drops the guest from the pool and 403s the NEXT restore-test/DR (campaign-2 R2). This
|
|
// re-asserts it after such a restore.
|
|
//
|
|
// PVE semantics: `PUT /pools/{poolid}` with `vms` is ADDITIVE (a merge) — `delete=1` is required to
|
|
// REMOVE, so passing a single vmid adds it without disturbing existing members. Adding a guest that
|
|
// is already a member is treated as a no-op success (idempotent): PVE reports "already" in the error
|
|
// body, which we swallow. Requires Pool.Allocate at /pool/<pool> (the token has it).
|
|
func (c *Client) PoolAddVMID(ctx context.Context, pool string, vmid int) error {
|
|
if pool == "" || vmid == 0 {
|
|
return fmt.Errorf("proxmox: PoolAddVMID needs pool and vmid")
|
|
}
|
|
v := url.Values{}
|
|
v.Set("vms", strconv.Itoa(vmid))
|
|
path := "/pools/" + url.PathEscape(pool)
|
|
_, err := c.dataString(ctx, http.MethodPut, path, v)
|
|
if err != nil {
|
|
// Idempotent: a guest already in the pool is success, not a failure.
|
|
if ae, ok := err.(*APIError); ok && strings.Contains(strings.ToLower(ae.Body), "already") {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ResizeLXC grows a guest volume via PUT /nodes/{node}/lxc/{vmid}/resize
|
|
// (token-covered: VM.Config.Disk + Datastore.AllocateSpace). Returns the UPID.
|
|
//
|
|
// disk is the volume key (e.g. "rootfs", "mp0"); size is a Proxmox size string. A
|
|
// LEADING '+' means GROW BY that amount (e.g. "+5G"); an absolute value can only ever
|
|
// grow (Proxmox rejects a shrink for a mounted/most volumes, but the agent must NOT rely
|
|
// on that — the reconcile layer is responsible for refusing a shrink before it reaches
|
|
// here, since a data-losing shrink is a destructive op, not a benign resize).
|
|
func (c *Client) ResizeLXC(ctx context.Context, vmid int, disk, size string) (string, error) {
|
|
if vmid == 0 || disk == "" || size == "" {
|
|
return "", fmt.Errorf("proxmox: ResizeLXC needs vmid, disk and size")
|
|
}
|
|
v := url.Values{}
|
|
v.Set("disk", disk)
|
|
v.Set("size", size)
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/resize", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodPut, path, v)
|
|
}
|
|
|
|
// Start starts a guest via POST /nodes/{node}/lxc/{vmid}/status/start (VM.PowerMgmt).
|
|
func (c *Client) Start(ctx context.Context, vmid int) (string, error) {
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/status/start", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodPost, path, url.Values{})
|
|
}
|
|
|
|
// Stop stops a guest via POST /nodes/{node}/lxc/{vmid}/status/stop (VM.PowerMgmt).
|
|
func (c *Client) Stop(ctx context.Context, vmid int) (string, error) {
|
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/status/stop", c.node, vmid)
|
|
return c.dataString(ctx, http.MethodPost, path, url.Values{})
|
|
}
|