ca0b169a4e
Pool membership is what lets the pool-scoped token reach a guest; pct restore --pool sets it only at CREATE, so a restore over an existing VMID drops the guest from the felhom pool and 403s the next restore-test/DR on VM.Audit. This empty-pool state is the true root cause of the campaign's "R1" (bind-mount restore failing was a symptom — restore-test's existing bind neutralization never ran without config-read). Add Client.PoolAddVMID (PUT /pools, additive+idempotent, Pool.Allocate) and call it in bring-up after liveness when spec.Pool!="" — warn-not-fail on a hiccup (liveness wins). B3 scratch-teardown 403 diagnosed as a cascade (restoretest already passes Pool). Role/ACL untouched. Tests + red-proof. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
208 lines
9.1 KiB
Go
208 lines
9.1 KiB
Go
package reconcile
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// RunState is a guest's desired/actual power state. The empty value means
|
|
// "unmanaged" on the desired side (the reconciler leaves run-state alone).
|
|
type RunState string
|
|
|
|
const (
|
|
// RunUnspecified (the zero value) — on a DesiredGuest it means run-state is not
|
|
// managed; the reconciler never starts/stops the guest for run-state reasons.
|
|
RunUnspecified RunState = ""
|
|
// RunRunning maps to proxmox status "running".
|
|
RunRunning RunState = "running"
|
|
// RunStopped maps to proxmox status "stopped".
|
|
RunStopped RunState = "stopped"
|
|
)
|
|
|
|
// normRun maps a raw proxmox status string to a RunState, collapsing anything
|
|
// unrecognized (e.g. "") to RunUnspecified so actual-state comparison is well-defined.
|
|
func normRun(status string) RunState {
|
|
switch status {
|
|
case "running":
|
|
return RunRunning
|
|
case "stopped":
|
|
return RunStopped
|
|
default:
|
|
return RunUnspecified
|
|
}
|
|
}
|
|
|
|
// DesiredGuest is the target state for one existing guest. Every field is
|
|
// individually optional ("unmanaged") so a desired-state source can pin only what it
|
|
// cares about — slice 4's planner only acts on the fields that are set.
|
|
type DesiredGuest struct {
|
|
VMID int
|
|
// Run is the target power state; RunUnspecified leaves it alone.
|
|
Run RunState
|
|
// Spec, when non-nil, manages sizing. Reuses hub.GuestSpec (cores/memory/disk).
|
|
// Phase A reconciles Cores and Memory via SetConfig; DiskBytes is reported but
|
|
// NOT reconciled here (a rootfs grow is `pct resize`, grow-only and separate —
|
|
// deferred to a later slice). Nil = sizing unmanaged.
|
|
Spec *hub.GuestSpec
|
|
// Description, when non-nil, manages the cosmetic `description` field (the first
|
|
// proven SetConfig round-trip, slice-4 pre-check). Nil = unmanaged.
|
|
Description *string
|
|
// Decommission, when true, is an EXPLICIT destructive intent to tear the guest down
|
|
// (slice 10A). It is the canonical destructive desired-state delta: the planner emits an
|
|
// ActionDecommission, which classifies ClassDecommission → Destructive → the gate refuses it
|
|
// `pending_signature` unless a verified operator signature is present. 10A never has a signer,
|
|
// so a decommission is always gated (never executed); the signed execution path is 10B. An
|
|
// explicit flag (not "absent from the desired list") is the safe design — a partial/empty hub
|
|
// list can never silently mass-destroy guests.
|
|
Decommission bool
|
|
}
|
|
|
|
// DesiredState is the vmid-keyed target for this host. At slice 4 the only live
|
|
// source is the empty provider, so Guests is empty in production; fixtures inject it
|
|
// in tests. Host-level desired state (storage manifest, etc.) arrives in later slices.
|
|
type DesiredState struct {
|
|
Guests map[int]DesiredGuest
|
|
}
|
|
|
|
// ActualGuest is one guest's observed state, read from Proxmox.
|
|
type ActualGuest struct {
|
|
VMID int
|
|
Run RunState
|
|
// SpecKnown is false when GuestConfig could not be read (the run-state from the
|
|
// list is still trusted; spec/description comparisons are skipped). Mirrors the
|
|
// collector's "keep run-status, omit spec" degradation.
|
|
SpecKnown bool
|
|
Cores int
|
|
MemoryMiB int64 // proxmox LXC `memory` is MiB
|
|
DiskBytes int64 // rootfs size in bytes (from the LXC list MaxDisk; for grow planning)
|
|
Description string // raw (may carry PVE's trailing newline; compared via normalizers)
|
|
}
|
|
|
|
// ActualState is the vmid-keyed observed state for this host.
|
|
type ActualState struct {
|
|
Guests map[int]ActualGuest
|
|
}
|
|
|
|
// DesiredProvider is the seam the desired-state source plugs into. At slice 4 the
|
|
// only implementation is EmptyProvider (no live source); slice 10's hub-serving path
|
|
// is the real implementation. Do NOT invent a hub/local-file source here.
|
|
type DesiredProvider interface {
|
|
Desired(ctx context.Context) (DesiredState, error)
|
|
}
|
|
|
|
// EmptyProvider is the slice-4 production provider: no desired state, so reconcile is
|
|
// a live no-op (the engine computes an empty action set).
|
|
type EmptyProvider struct{}
|
|
|
|
// Desired returns an empty desired state.
|
|
func (EmptyProvider) Desired(context.Context) (DesiredState, error) {
|
|
return DesiredState{Guests: map[int]DesiredGuest{}}, nil
|
|
}
|
|
|
|
// StaticProvider serves a fixed DesiredState — used by fixtures (and usable as a
|
|
// local override later). It never mutates the value it was given.
|
|
type StaticProvider struct{ State DesiredState }
|
|
|
|
// Desired returns the static state.
|
|
func (p StaticProvider) Desired(context.Context) (DesiredState, error) { return p.State, nil }
|
|
|
|
// CachingProvider is the slice-10A production provider: a thread-safe cache of the hub-served
|
|
// DesiredState plus the generation it corresponds to. The hub-sync layer (internal/desired) calls
|
|
// Update when the heartbeat envelope's generation advances and a fresh fetch arrives; the engine
|
|
// reads the cache via Desired each reconcile tick. Until the first Update it returns an empty
|
|
// state (generation 0) — so reconcile is a live no-op exactly like EmptyProvider, with zero
|
|
// mutations, which is the correct cold-start behaviour.
|
|
type CachingProvider struct {
|
|
mu sync.RWMutex
|
|
state DesiredState
|
|
gen int64
|
|
}
|
|
|
|
// NewCachingProvider builds an empty provider (generation 0, no guests).
|
|
func NewCachingProvider() *CachingProvider {
|
|
return &CachingProvider{state: DesiredState{Guests: map[int]DesiredGuest{}}}
|
|
}
|
|
|
|
// Desired returns the cached state (a shallow copy of the guest map so a caller can't mutate the
|
|
// cache, and a concurrent Update can't race the read).
|
|
func (p *CachingProvider) Desired(context.Context) (DesiredState, error) {
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
out := DesiredState{Guests: make(map[int]DesiredGuest, len(p.state.Guests))}
|
|
for k, v := range p.state.Guests {
|
|
out.Guests[k] = v
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Update replaces the cached state + generation (called by the sync layer on a generation advance).
|
|
func (p *CachingProvider) Update(generation int64, state DesiredState) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
if state.Guests == nil {
|
|
state.Guests = map[int]DesiredGuest{}
|
|
}
|
|
p.state = state
|
|
p.gen = generation
|
|
}
|
|
|
|
// Generation returns the cached generation (the agent's view of "what I have applied from").
|
|
func (p *CachingProvider) Generation() int64 {
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.gen
|
|
}
|
|
|
|
// GuestAPI is the narrow Proxmox surface the engine needs: read actual state and
|
|
// dispatch the benign-on-existing-guest mutations. *proxmox.Client satisfies it; a
|
|
// fake satisfies it in tests. Every mutating call returns a UPID (or "" for the
|
|
// synchronous path) per the proxmox/mutate.go contract — the engine WaitTasks a
|
|
// non-empty UPID and treats "" as a clean synchronous success.
|
|
type GuestAPI interface {
|
|
ListLXC(ctx context.Context) ([]proxmox.Guest, error)
|
|
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
|
Start(ctx context.Context, vmid int) (string, error)
|
|
Stop(ctx context.Context, vmid int) (string, error)
|
|
SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error)
|
|
// ResizeLXC grows a volume (grow-only; the planner never emits a shrink). Async → UPID.
|
|
ResizeLXC(ctx context.Context, vmid int, disk, size string) (string, error)
|
|
// RestoreLXC restores an archive into a (fresh) vmid — the create path (slice 6). Async → UPID.
|
|
RestoreLXC(ctx context.Context, opts proxmox.RestoreLXCOptions) (string, error)
|
|
// PoolAddVMID re-asserts pool membership after a restore-over-existing (campaign-2 R2). Sync (no
|
|
// UPID); idempotent. Membership is what lets the pool-scoped token reach the guest next time.
|
|
PoolAddVMID(ctx context.Context, pool string, vmid int) error
|
|
// DestroyLXC destroys a guest — the scratch-teardown primitive (slice 6). Async → UPID.
|
|
// Destructive-class; the engine only ever issues it for an agent-tagged scratch guest
|
|
// (benign by provenance) via the gate.
|
|
DestroyLXC(ctx context.Context, vmid int) (string, error)
|
|
// GuestStatus reads a single guest's current status (run-state poll during a restore-test).
|
|
GuestStatus(ctx context.Context, vmid int) (proxmox.Guest, error)
|
|
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
|
// TaskStatusOnce is a single non-blocking task-status read — used by crash
|
|
// recovery to learn the outcome of an op that was in flight when the agent died.
|
|
TaskStatusOnce(ctx context.Context, upid string) (proxmox.TaskStatus, error)
|
|
// TaskLogTail fetches up to limit trailing task-log lines — used to surface a guest-start
|
|
// task's warning text (the restore-test fetches it when the start exits "WARNINGS: N", so
|
|
// the advisory is reported without failing a guest that actually boots).
|
|
TaskLogTail(ctx context.Context, upid string, limit int) ([]string, error)
|
|
}
|
|
|
|
// guestDescription decodes the (string-valued) `description` key from a GuestConfig's
|
|
// raw Extra map, returning "" when absent. The value is returned raw — PVE appends a
|
|
// trailing newline on read, which the normalization layer strips at comparison time.
|
|
func guestDescription(cfg proxmox.GuestConfig) string {
|
|
raw, ok := cfg.Extra["description"]
|
|
if !ok || len(raw) == 0 {
|
|
return ""
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
return ""
|
|
}
|
|
return s
|
|
}
|