slice 10A: activate the control envelope (Down channel) + hub-backed desired provider (v0.15.0)
The control envelope becomes live: the agent caches the hub's desired-state +
generation and re-fetches GET /hosts/{id}/desired-state only when the
generation advances. A new internal/desired Syncer maps the wire shape into a
reconcile.CachingProvider feeding the engine; benign deltas reconcile, an
explicit guest decommission is gated pending_signature (exec is 10B). Adds the
DesiredStateResponse/WireDesiredState wire types + Client.FetchDesiredState +
the loop EnvelopeObserver seam. Cross-repo golden (envelope + desired-state)
byte-identical with the hub.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -107,6 +107,8 @@ func classOfAction(k ActionKind) OpClass {
|
||||
return ClassSetConfig
|
||||
case ActionResize:
|
||||
return ClassResize
|
||||
case ActionDecommission:
|
||||
return ClassDecommission
|
||||
default:
|
||||
return OpClass(k)
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ type Result struct {
|
||||
Planned int
|
||||
Executed int // succeeded
|
||||
Failed int // errored
|
||||
Pending int // destructive actions gated pending_signature (slice 10A — expected, not failed)
|
||||
Errors []error // one per failed action
|
||||
}
|
||||
|
||||
@@ -110,11 +111,12 @@ func (e *Engine) Reconcile(ctx context.Context) (Result, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Every mutation passes the reversibility gate before the queue (doc 03 §4).
|
||||
// Reconcile only produces benign actions, so each is allowed unsigned — but the
|
||||
// gate is genuinely in the path: a destructive class here would be refused
|
||||
// (pending_signature) and never dispatched. A gate refusal counts as a failed
|
||||
// action (it should not happen for the benign reconcile set).
|
||||
// Every mutation passes the reversibility gate before the queue (doc 03 §4). Benign actions
|
||||
// are allowed unsigned; a DESTRUCTIVE delta (slice 10A: an explicit decommission) is refused
|
||||
// `pending_signature` when no operator signature is present — that is EXPECTED, not a failure:
|
||||
// 10A serves destructive intent but never executes it (the signed-op execution is 10B). So a
|
||||
// pending_signature refusal is counted as Pending and logged at INFO; any OTHER refusal (a
|
||||
// benign action denied, or a destructive one rejected for a different reason) is a real failure.
|
||||
type dispatched struct {
|
||||
act Action
|
||||
ch <-chan error
|
||||
@@ -124,10 +126,16 @@ func (e *Engine) Reconcile(ctx context.Context) (Result, error) {
|
||||
act := actions[i]
|
||||
dec := e.gate.Authorize(intentForAction(e.hostID, act), nil)
|
||||
if !dec.Allowed {
|
||||
if dec.Disposition == Destructive && dec.Reason == ReasonPendingSignature {
|
||||
res.Pending++
|
||||
e.logger.Info("reconcile: destructive action gated pending operator signature (slice 10B)",
|
||||
"vmid", act.VMID, "kind", act.Kind, "reason", dec.Reason)
|
||||
continue
|
||||
}
|
||||
res.Failed++
|
||||
res.Errors = append(res.Errors, fmt.Errorf("reconcile: gate refused %s vmid %d: %s",
|
||||
act.Kind, act.VMID, dec.Reason))
|
||||
e.logger.Error("reconcile: gate refused a benign action (unexpected)",
|
||||
e.logger.Error("reconcile: gate refused an action unexpectedly",
|
||||
"vmid", act.VMID, "kind", act.Kind, "reason", dec.Reason)
|
||||
continue
|
||||
}
|
||||
@@ -174,6 +182,12 @@ func (e *Engine) execute(ctx context.Context, act Action) error {
|
||||
} else {
|
||||
upid, err = e.api.ResizeLXC(ctx, act.VMID, disk, size)
|
||||
}
|
||||
case ActionDecommission:
|
||||
// Reaching here means a destructive decommission passed the gate (a verified signature) —
|
||||
// which only happens once 10B wires the signed-op executor. In 10A there is no signer, so
|
||||
// the gate refuses it before dispatch and this branch is unreachable. Fail safe loudly
|
||||
// rather than silently no-op, so a future signed path can't accidentally execute here.
|
||||
err = fmt.Errorf("reconcile: decommission executor is slice 10B (refusing to execute vmid %d)", act.VMID)
|
||||
default:
|
||||
err = fmt.Errorf("reconcile: unknown action kind %q", act.Kind)
|
||||
}
|
||||
@@ -261,7 +275,7 @@ func (e *Engine) reconcileOnce(ctx context.Context) {
|
||||
}
|
||||
if res.Planned > 0 {
|
||||
e.logger.Info("reconcile: pass complete",
|
||||
"planned", res.Planned, "executed", res.Executed, "failed", res.Failed)
|
||||
"planned", res.Planned, "executed", res.Executed, "failed", res.Failed, "pending", res.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,11 @@ const (
|
||||
// emits it only when desired DiskBytes > actual; a shrink is data-losing and is refused
|
||||
// (never silently applied as a grow). Slice 5 Phase B; unfed live until slice 10.
|
||||
ActionResize ActionKind = "resize"
|
||||
// ActionDecommission tears a guest down — the canonical DESTRUCTIVE delta (slice 10A). The
|
||||
// planner emits it for an explicit DesiredGuest.Decommission; it classifies ClassDecommission
|
||||
// → Destructive, so the gate refuses it `pending_signature` (no signer in 10A → never
|
||||
// executed). Its EXECUTOR is slice 10B; 10A only plans + gates it.
|
||||
ActionDecommission ActionKind = "decommission"
|
||||
)
|
||||
|
||||
// growRoundMiB rounds a positive byte delta UP to whole MiB for the Proxmox `+<n>M` grow
|
||||
@@ -88,6 +93,20 @@ func Plan(desired DesiredState, actual ActualState, norm FieldNormalizers) []Act
|
||||
continue
|
||||
}
|
||||
|
||||
// EXPLICIT decommission (slice 10A) — the destructive delta. Emit it as a single
|
||||
// ActionDecommission and emit NOTHING else for this guest (no point reconciling cores
|
||||
// on a guest the operator wants torn down). It is classified Destructive downstream, so
|
||||
// the gate refuses it pending_signature in 10A (executor is 10B). Only emitted when the
|
||||
// guest actually exists (decommissioning an absent guest is a no-op).
|
||||
if d.Decommission {
|
||||
actions = append(actions, Action{
|
||||
VMID: vmid,
|
||||
Kind: ActionDecommission,
|
||||
Reason: "decommission requested (destructive — requires operator signature)",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Benign spec/description changes → a single SetConfig, only when we could
|
||||
// read the current config (else we'd write blind).
|
||||
if a.SpecKnown {
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// The headline slice-10A reconcile behaviour: a desired-state carrying ONE benign delta and ONE
|
||||
// destructive (decommission) delta → the benign op is applied, the destructive one is GATED
|
||||
// pending_signature (counted Pending, NOT Failed) and is NEVER executed (no signer in 10A).
|
||||
func TestReconcile_BenignAppliedDestructiveGated(t *testing.T) {
|
||||
api := &fakeAPI{
|
||||
lxc: []proxmox.Guest{
|
||||
{VMID: 100, Status: "stopped"}, // benign: desired running → Start
|
||||
{VMID: 200, Status: "running"}, // destructive: decommission → gated
|
||||
},
|
||||
cfg: map[int]proxmox.GuestConfig{100: {Cores: 2}, 200: {Cores: 2}},
|
||||
}
|
||||
provider := StaticProvider{State: DesiredState{Guests: map[int]DesiredGuest{
|
||||
100: {VMID: 100, Run: RunRunning},
|
||||
200: {VMID: 200, Decommission: true},
|
||||
}}}
|
||||
e, _, _ := newEngine(t, api, provider)
|
||||
|
||||
res, err := e.Reconcile(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile: %v", err)
|
||||
}
|
||||
if res.Planned != 2 {
|
||||
t.Errorf("planned = %d, want 2 (one benign + one destructive)", res.Planned)
|
||||
}
|
||||
// Benign Start(100) applied.
|
||||
if res.Executed != 1 || len(api.starts) != 1 || api.starts[0] != 100 {
|
||||
t.Errorf("benign delta not applied: executed=%d starts=%v", res.Executed, api.starts)
|
||||
}
|
||||
// Destructive decommission GATED pending (not a failure).
|
||||
if res.Pending != 1 {
|
||||
t.Errorf("pending = %d, want 1 (decommission gated pending_signature)", res.Pending)
|
||||
}
|
||||
if res.Failed != 0 {
|
||||
t.Errorf("failed = %d, want 0 (a pending_signature gate is expected, not a failure)", res.Failed)
|
||||
}
|
||||
// And it was NEVER executed: no destroy/decommission op reached Proxmox.
|
||||
if len(api.destroys) != 0 {
|
||||
t.Errorf("destructive decommission EXECUTED (destroys=%v) — it must be gated, not run", api.destroys)
|
||||
}
|
||||
}
|
||||
|
||||
// Plan unit: an explicit Decommission emits exactly one ActionDecommission and suppresses any
|
||||
// other delta for that guest (no point reconciling cores on a guest being torn down).
|
||||
func TestPlan_DecommissionEmitsDestructiveActionOnly(t *testing.T) {
|
||||
desired := DesiredState{Guests: map[int]DesiredGuest{
|
||||
// Decommission set AND a spec drift — only the decommission should be emitted.
|
||||
7: {VMID: 7, Decommission: true, Spec: &hub.GuestSpec{Cores: 9, MemoryBytes: 9 << 20}, Run: RunStopped},
|
||||
}}
|
||||
actual := ActualState{Guests: map[int]ActualGuest{
|
||||
7: {VMID: 7, Run: RunRunning, SpecKnown: true, Cores: 2},
|
||||
}}
|
||||
actions := Plan(desired, actual, DefaultNormalizers())
|
||||
if len(actions) != 1 {
|
||||
t.Fatalf("actions = %d (%+v), want exactly 1 (decommission only)", len(actions), actions)
|
||||
}
|
||||
if actions[0].Kind != ActionDecommission || actions[0].VMID != 7 {
|
||||
t.Errorf("action = %+v, want decommission of vmid 7", actions[0])
|
||||
}
|
||||
// And it classifies destructive.
|
||||
if Classify(classOfAction(ActionDecommission), Provenance{}) != Destructive {
|
||||
t.Error("ActionDecommission must classify Destructive (no provenance)")
|
||||
}
|
||||
}
|
||||
|
||||
// Decommission of an ABSENT guest is a no-op (nothing to tear down).
|
||||
func TestPlan_DecommissionAbsentGuestNoop(t *testing.T) {
|
||||
desired := DesiredState{Guests: map[int]DesiredGuest{7: {VMID: 7, Decommission: true}}}
|
||||
actual := ActualState{Guests: map[int]ActualGuest{}} // guest 7 not present
|
||||
if actions := Plan(desired, actual, DefaultNormalizers()); len(actions) != 0 {
|
||||
t.Errorf("decommission of absent guest emitted %+v, want none", actions)
|
||||
}
|
||||
}
|
||||
|
||||
// CachingProvider: empty until Update, then serves the cached state + generation, and isolates
|
||||
// the cache from caller mutation.
|
||||
func TestCachingProvider_UpdateAndIsolation(t *testing.T) {
|
||||
p := NewCachingProvider()
|
||||
if p.Generation() != 0 {
|
||||
t.Fatalf("fresh generation = %d, want 0", p.Generation())
|
||||
}
|
||||
if st, _ := p.Desired(context.Background()); len(st.Guests) != 0 {
|
||||
t.Fatalf("fresh provider should be empty, got %+v", st.Guests)
|
||||
}
|
||||
|
||||
p.Update(3, DesiredState{Guests: map[int]DesiredGuest{5: {VMID: 5, Run: RunRunning}}})
|
||||
if p.Generation() != 3 {
|
||||
t.Errorf("generation after update = %d, want 3", p.Generation())
|
||||
}
|
||||
st, _ := p.Desired(context.Background())
|
||||
if len(st.Guests) != 1 || st.Guests[5].Run != RunRunning {
|
||||
t.Fatalf("cached state = %+v", st.Guests)
|
||||
}
|
||||
// Mutating the returned copy must NOT affect the cache.
|
||||
st.Guests[5] = DesiredGuest{VMID: 5, Run: RunStopped}
|
||||
st.Guests[99] = DesiredGuest{VMID: 99}
|
||||
st2, _ := p.Desired(context.Background())
|
||||
if len(st2.Guests) != 1 || st2.Guests[5].Run != RunRunning {
|
||||
t.Errorf("cache was mutated by a caller: %+v", st2.Guests)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package reconcile
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
@@ -50,6 +51,14 @@ type DesiredGuest struct {
|
||||
// Description, when non-nil, manages the cosmetic `description` field (the first
|
||||
// proven SetConfig round-trip, slice-4 pre-check). Nil = unmanaged.
|
||||
Description *string
|
||||
// Decommission, when true, is an EXPLICIT destructive intent to tear the guest down
|
||||
// (slice 10A). It is the canonical destructive desired-state delta: the planner emits an
|
||||
// ActionDecommission, which classifies ClassDecommission → Destructive → the gate refuses it
|
||||
// `pending_signature` unless a verified operator signature is present. 10A never has a signer,
|
||||
// so a decommission is always gated (never executed); the signed execution path is 10B. An
|
||||
// explicit flag (not "absent from the desired list") is the safe design — a partial/empty hub
|
||||
// list can never silently mass-destroy guests.
|
||||
Decommission bool
|
||||
}
|
||||
|
||||
// DesiredState is the vmid-keyed target for this host. At slice 4 the only live
|
||||
@@ -101,6 +110,53 @@ type StaticProvider struct{ State DesiredState }
|
||||
// Desired returns the static state.
|
||||
func (p StaticProvider) Desired(context.Context) (DesiredState, error) { return p.State, nil }
|
||||
|
||||
// CachingProvider is the slice-10A production provider: a thread-safe cache of the hub-served
|
||||
// DesiredState plus the generation it corresponds to. The hub-sync layer (internal/desired) calls
|
||||
// Update when the heartbeat envelope's generation advances and a fresh fetch arrives; the engine
|
||||
// reads the cache via Desired each reconcile tick. Until the first Update it returns an empty
|
||||
// state (generation 0) — so reconcile is a live no-op exactly like EmptyProvider, with zero
|
||||
// mutations, which is the correct cold-start behaviour.
|
||||
type CachingProvider struct {
|
||||
mu sync.RWMutex
|
||||
state DesiredState
|
||||
gen int64
|
||||
}
|
||||
|
||||
// NewCachingProvider builds an empty provider (generation 0, no guests).
|
||||
func NewCachingProvider() *CachingProvider {
|
||||
return &CachingProvider{state: DesiredState{Guests: map[int]DesiredGuest{}}}
|
||||
}
|
||||
|
||||
// Desired returns the cached state (a shallow copy of the guest map so a caller can't mutate the
|
||||
// cache, and a concurrent Update can't race the read).
|
||||
func (p *CachingProvider) Desired(context.Context) (DesiredState, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
out := DesiredState{Guests: make(map[int]DesiredGuest, len(p.state.Guests))}
|
||||
for k, v := range p.state.Guests {
|
||||
out.Guests[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Update replaces the cached state + generation (called by the sync layer on a generation advance).
|
||||
func (p *CachingProvider) Update(generation int64, state DesiredState) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if state.Guests == nil {
|
||||
state.Guests = map[int]DesiredGuest{}
|
||||
}
|
||||
p.state = state
|
||||
p.gen = generation
|
||||
}
|
||||
|
||||
// Generation returns the cached generation (the agent's view of "what I have applied from").
|
||||
func (p *CachingProvider) Generation() int64 {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.gen
|
||||
}
|
||||
|
||||
// GuestAPI is the narrow Proxmox surface the engine needs: read actual state and
|
||||
// dispatch the benign-on-existing-guest mutations. *proxmox.Client satisfies it; a
|
||||
// fake satisfies it in tests. Every mutating call returns a UPID (or "" for the
|
||||
|
||||
Reference in New Issue
Block a user