admin/felhom-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.60.0: proof-of-launch destroy gating (F1a/b/c) + restore-test band-advance (F2)

Browse Source 
Campaign pool-effects F1 (HIGH): the bring-up compensating rollback and the
restore-test teardown destroyed the target vmid even when RestoreLXC failed
synchronously without creating anything — destroying a guest the transaction
never made (only the pool ACL 403 contained it). A RestoreLXC UPID is now the
sole destroy authorization in all three destroy paths (in-process bring-up
defer, in-process restore-test teardown, Recover). F2: the restore-test
advances past an 'already exists' band vmid (invisible squatter) instead of
failing + false-alerting; a fully-occupied band Skips.

Red-proof verified: with the gates reverted, the four new tests fail with the
innocent-guest destroy. go build/vet/test clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
authentik Default Admin
2026-07-02 10:18:51 +02:00
parent 55ade9e254
commit b9356d60ab
8 changed files with 400 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/reconcile/bringup_test.go
+81 -5
View File
@@ -255,7 +255,17 @@ func TestRunBringUp_CompensatingRollback(t *testing.T) {
name string
setup func(*fakeAPI)
}{
{"restore error", func(a *fakeAPI) { a.restoreErr = errors.New("restore boom") }},
{"restore-task failure after launch", func(a *fakeAPI) {
// Scenario B (no-regression): the restore POST was ACCEPTED (UPID) but the task then
// fails → we created/started it → the compensating destroy MUST still fire.
a.restoreUPID = "UPID:demo:restore:8000:"
a.waitFunc = func(upid string) (proxmox.TaskStatus, error) {
if upid == "UPID:demo:restore:8000:" {
return proxmox.TaskStatus{}, errors.New("restore task failed")
}
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
}
}},
{"config real error", func(a *fakeAPI) {
a.setFunc = func(int, map[string]string) (string, error) {
return "", &proxmox.APIError{StatusCode: 500, Body: "some non-lock internal error"}
@@ -300,6 +310,48 @@ func TestRunBringUp_CompensatingRollback(t *testing.T) {
}
}
// TestRunBringUp_NoLaunchNoDestroy is the F1a red-proof (campaign pool-effects): a restore that
// fails SYNCHRONOUSLY (no UPID — nothing created) must NOT arm the compensating destroy. The
// headline case is PVE refusing a vmid that already holds a guest the pool-blind duplicate guard
// couldn't see — the old defer would have destroyed that innocent pre-existing guest (only the
// pool ACL's 403 saved the non-pool subset; an in-pool one would have been destroyed).
// Red-proof: revert the `launched` gate in runBringUp and this test fails with destroys=[8000].
func TestRunBringUp_NoLaunchNoDestroy(t *testing.T) {
const vmid = 8000
cases := []struct {
name string
err error
}{
{"PVE refuses pre-existing vmid", &proxmox.APIError{
StatusCode: 500, Method: "POST", Path: "/nodes/x/lxc",
Body: `{"message":"CT 8000 already exists on node 'x'\n","data":null}`,
}},
{"plain synchronous restore error", errors.New("restore boom")},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}, restoreErr: tc.err}
e, j, q := newEngine(t, api, EmptyProvider{})
defer q.Close()
res := e.RunBringUp(context.Background(), BringUpSpec{
Mode: ModeProvision, Archive: "vol", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "h",
})
if res.Pass || res.Err == nil {
t.Fatalf("must fail, got %+v", res)
}
// THE point: no destroy is even attempted — the txn created nothing.
if len(api.destroys) != 0 {
t.Fatalf("no-launch failure must NOT destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
}
// And the owning entry is closed terminal in-process (not left for Recover).
if len(j.InFlight()) != 0 {
t.Errorf("owning entry must be terminal after a no-launch failure: %+v", j.InFlight())
}
})
}
}
func TestRunBringUp_DRPreservesContinuityIdentity(t *testing.T) {
const vmid = 8001
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
@@ -513,10 +565,11 @@ func TestRunBringUp_RejectsReservedAndExistingVMID(t *testing.T) {
func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) {
const vmid = 8000
// The guest still exists at startup (agent crashed mid-bring-up) → Recover destroys it.
// The guest still exists at startup (agent crashed mid-bring-up AFTER the restore launched —
// the entry carries the UPID = the launch proof) → Recover destroys it.
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "running"}}}
e, j, _ := newEngine(t, api, EmptyProvider{})
if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
if err := j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
res := e.Recover(context.Background())
@@ -533,10 +586,10 @@ func TestRecover_HalfBuiltBringUpRolledBack(t *testing.T) {
func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) {
const vmid = 8000
// Crash after the restore POST failed (no guest) → idempotent clean, no destroy.
// Crash after a LAUNCHED restore whose guest is already gone → idempotent clean, no destroy.
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001}}} // 8000 absent
e, j, _ := newEngine(t, api, EmptyProvider{})
j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()})
j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, UPID: "UPID:demo:restore:8000:", State: OpTaskRunning, At: time.Now().UTC()})
res := e.Recover(context.Background())
if res.BringUpClean != 1 || len(api.destroys) != 0 {
t.Fatalf("already-gone bring-up must be clean with no destroy: res=%+v destroys=%+v", res, api.destroys)
@@ -546,6 +599,29 @@ func TestRecover_HalfBuiltBringUpAlreadyGone(t *testing.T) {
}
}
// TestRecover_BringUpNoUPIDAbandoned is the F1c red-proof (campaign pool-effects, Scenario C): a
// crash left a Rollback entry with NO UPID (the restore was never confirmed) while a guest sits
// at that vmid — e.g. a PRE-EXISTING guest the pool-blind list-existence check would have called
// "ours". Recover must ABANDON (fail-safe), never destroy: no journaled UPID ⇒ this transaction
// created nothing. Red-proof: route the entry to recoverBringUp (pre-fix order) and this fails
// with destroys=[8000].
func TestRecover_BringUpNoUPIDAbandoned(t *testing.T) {
const vmid = 8000
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: vmid, Status: "stopped"}}} // a guest IS at the vmid
e, j, _ := newEngine(t, api, EmptyProvider{})
j.Append(JournalEntry{OpID: "bring-up-8000-1", VMID: vmid, Kind: bringUpKind, Rollback: true, State: OpStarted, At: time.Now().UTC()})
res := e.Recover(context.Background())
if len(api.destroys) != 0 {
t.Fatalf("a no-UPID Rollback entry must NEVER destroy the vmid (pre-existing guest!): destroys=%+v", api.destroys)
}
if res.RolledBack != 1 || res.BringUpRolledBack != 0 {
t.Fatalf("entry must be abandoned via the no-UPID fail-safe path, got %+v", res)
}
if len(j.InFlight()) != 0 {
t.Errorf("abandoned entry must be terminal: %+v", j.InFlight())
}
}
// lockBackoffFast shrinks the F4 retry backoff for tests and restores it after.
func lockBackoffFast(t *testing.T) {
t.Helper()