v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)
The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→ teardown) that closes "a backup you haven't restored isn't a backup". All benign (reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent. - proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID. - reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent). - internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence scheduler (4th daemon goroutine, default 24h) + in-memory report store. - hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical + bidirectional key-set tests; hub handler logs a FAILED restore-test prominently. - config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,18 +20,24 @@ type fakeAPI struct {
|
||||
|
||||
startUPID, stopUPID, setUPID, resizeUPID string
|
||||
startErr, stopErr, setErr, resizeErr error
|
||||
restoreUPID, destroyUPID string
|
||||
restoreErr, destroyErr error
|
||||
// status maps vmid -> the Guest returned by GuestStatus (default running if absent).
|
||||
status map[int]proxmox.Guest
|
||||
// waitFunc maps a UPID to a (status, err); default = OK. Mirrors the real client,
|
||||
// which errors on a non-OK exitstatus.
|
||||
waitFunc func(upid string) (proxmox.TaskStatus, error)
|
||||
// statusFunc backs TaskStatusOnce (crash recovery); default = stopped/OK.
|
||||
statusFunc func(upid string) (proxmox.TaskStatus, error)
|
||||
|
||||
starts []int
|
||||
stops []int
|
||||
sets []setCall
|
||||
resizes []resizeCall
|
||||
waits []string
|
||||
listErr error
|
||||
starts []int
|
||||
stops []int
|
||||
sets []setCall
|
||||
resizes []resizeCall
|
||||
restores []proxmox.RestoreLXCOptions
|
||||
destroys []int
|
||||
waits []string
|
||||
listErr error
|
||||
}
|
||||
|
||||
type resizeCall struct {
|
||||
@@ -39,6 +45,29 @@ type resizeCall struct {
|
||||
disk, size string
|
||||
}
|
||||
|
||||
func (f *fakeAPI) RestoreLXC(_ context.Context, opts proxmox.RestoreLXCOptions) (string, error) {
|
||||
f.mu.Lock()
|
||||
f.restores = append(f.restores, opts)
|
||||
f.mu.Unlock()
|
||||
return f.restoreUPID, f.restoreErr
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DestroyLXC(_ context.Context, vmid int) (string, error) {
|
||||
f.mu.Lock()
|
||||
f.destroys = append(f.destroys, vmid)
|
||||
f.mu.Unlock()
|
||||
return f.destroyUPID, f.destroyErr
|
||||
}
|
||||
|
||||
func (f *fakeAPI) GuestStatus(_ context.Context, vmid int) (proxmox.Guest, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if g, ok := f.status[vmid]; ok {
|
||||
return g, nil
|
||||
}
|
||||
return proxmox.Guest{VMID: vmid, Status: "running"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) TaskStatusOnce(_ context.Context, upid string) (proxmox.TaskStatus, error) {
|
||||
if f.statusFunc != nil {
|
||||
return f.statusFunc(upid)
|
||||
|
||||
@@ -45,7 +45,13 @@ type JournalEntry struct {
|
||||
UPID string `json:"upid,omitempty"`
|
||||
State OpState `json:"state"`
|
||||
IdempKey string `json:"idemp_key,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Scratch marks an entry that OWNS an agent-tagged scratch guest at VMID (slice 6
|
||||
// restore-test). While such an entry is in-flight, the load-bearing invariant is "the
|
||||
// scratch guest may exist and MUST be destroyed" — so Recover resolves it by ensuring
|
||||
// VMID is gone (a benign teardown), NOT by re-checking any sub-task UPID. The entry is
|
||||
// terminal only after teardown. See recover.go.
|
||||
Scratch bool `json:"scratch,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Journal is the durable operation log + idempotency store. It mirrors
|
||||
|
||||
@@ -3,6 +3,8 @@ package reconcile
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// Recover consumes the journal's in-flight set at startup: resume-or-rollback for any
|
||||
@@ -32,6 +34,16 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
|
||||
}
|
||||
for _, entry := range e.journal.InFlight() {
|
||||
res.Examined++
|
||||
|
||||
// Scratch entries (slice-6 restore-test) are resolved by TEARDOWN, not by
|
||||
// re-checking a sub-task UPID — a leaked scratch guest is the failure mode that
|
||||
// matters. Handle them BEFORE the generic UPID path (else the restore sub-task's OK
|
||||
// status would mark the entry succeeded while the guest still exists → leak).
|
||||
if entry.Scratch {
|
||||
e.recoverScratch(ctx, entry, &res)
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.UPID == "" {
|
||||
// POST never confirmed → abandon (fail-safe).
|
||||
e.append(terminal(entry, OpFailed))
|
||||
@@ -72,18 +84,80 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
|
||||
return res
|
||||
}
|
||||
|
||||
// recoverScratch resolves a leaked restore-test scratch guest (slice 6, doc 03 §8/§10).
|
||||
// The invariant: a Scratch entry in-flight at startup means "scratch guest VMID may exist
|
||||
// and must be destroyed." It is idempotent — if the guest is already gone (crash after the
|
||||
// destroy task but before the terminal record), it records terminal-clean. The teardown
|
||||
// routes through the gate as a benign ClassGuestDestroy (agent-tagged scratch provenance) —
|
||||
// the same audit-bearing path the normal teardown uses.
|
||||
func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *RecoverResult) {
|
||||
lxc, err := e.api.ListLXC(ctx)
|
||||
if err != nil {
|
||||
// Can't tell whether the guest exists → leave in-flight; a later Recover retries.
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: cannot list guests to resolve leaked scratch; left in-flight",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
exists := false
|
||||
for _, g := range lxc {
|
||||
if g.VMID == entry.VMID {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
// Already gone (idempotent) → the scratch left no leak.
|
||||
e.append(terminal(entry, OpSucceeded))
|
||||
res.ScratchClean++
|
||||
e.logger.Info("recover: leaked-scratch entry resolved; guest already gone",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID)
|
||||
return
|
||||
}
|
||||
|
||||
dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, entry.VMID), nil)
|
||||
if !dec.Allowed {
|
||||
// Should be benign; if not, fail-safe (leave in-flight, do NOT force a destroy).
|
||||
res.Unresolved++
|
||||
e.logger.Error("recover: scratch teardown refused by gate (unexpected); left in-flight",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
|
||||
return
|
||||
}
|
||||
upid, err := e.api.DestroyLXC(ctx, entry.VMID)
|
||||
if err != nil {
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: destroying leaked scratch failed; left in-flight (will retry)",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
||||
res.Unresolved++
|
||||
e.logger.Warn("recover: leaked-scratch destroy task failed; left in-flight (will retry)",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
e.append(terminal(entry, OpSucceeded))
|
||||
res.ScratchDestroyed++
|
||||
e.logger.Warn("recover: destroyed leaked restore-test scratch guest",
|
||||
"op_id", entry.OpID, "vmid", entry.VMID)
|
||||
}
|
||||
|
||||
// RecoverResult summarizes a startup recovery pass.
|
||||
type RecoverResult struct {
|
||||
Examined int
|
||||
Resumed int // task found completed OK and recorded succeeded
|
||||
Failed int // task found ended non-OK and recorded failed
|
||||
RolledBack int // no task id → abandoned (fail-safe)
|
||||
StillRunning int // task still executing → left in-flight
|
||||
Unresolved int // task status unreadable → left in-flight
|
||||
Examined int
|
||||
Resumed int // task found completed OK and recorded succeeded
|
||||
Failed int // task found ended non-OK and recorded failed
|
||||
RolledBack int // no task id → abandoned (fail-safe)
|
||||
StillRunning int // task still executing → left in-flight
|
||||
Unresolved int // task status unreadable → left in-flight
|
||||
ScratchClean int // scratch entry resolved: guest already gone (no leak)
|
||||
ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
|
||||
}
|
||||
|
||||
// terminal builds a terminal journal record preserving the op's identity, with the
|
||||
// idempotency key carried through so a SUCCEEDED one-shot op marks its key applied.
|
||||
// idempotency key + scratch flag carried through.
|
||||
func terminal(e JournalEntry, state OpState) JournalEntry {
|
||||
return JournalEntry{
|
||||
OpID: e.OpID,
|
||||
@@ -92,6 +166,7 @@ func terminal(e JournalEntry, state OpState) JournalEntry {
|
||||
UPID: e.UPID,
|
||||
State: state,
|
||||
IdempKey: e.IdempKey,
|
||||
Scratch: e.Scratch,
|
||||
At: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// The self-restore-test (doc 03 §8) — the piece that closes "a backup you haven't restored
|
||||
// isn't a backup". It is a JOURNALED reconcile job so it inherits the slice-4 journal,
|
||||
// per-guest serialization, and crash-safe recovery: a mid-test crash can't leak a scratch
|
||||
// guest (engine.Recover tears it down). Every step here is BENIGN — restore-to-new
|
||||
// (ClassCreate), a benign net-link-down SetConfig, and a scratch teardown that is benign by
|
||||
// agent-tagged-scratch provenance (no new destructive class, no new crypto).
|
||||
|
||||
// scratchKind is the journal Kind for a restore-test scratch-guest-owning entry. Recover
|
||||
// keys off JournalEntry.Scratch (not this string), but the Kind aids audit/debug.
|
||||
const scratchKind = "scratch_restore_test"
|
||||
|
||||
// DefaultBootTimeout bounds how long the restore-test waits for the scratch guest to reach
|
||||
// running before declaring the verify failed.
|
||||
const DefaultBootTimeout = 2 * time.Minute
|
||||
|
||||
// RestoreTestSpec parameterizes one restore-test.
|
||||
type RestoreTestSpec struct {
|
||||
Archive string // source archive volid to restore (resolved by the caller)
|
||||
SourceTier string // "local" this slice (pbs = Phase B) — for the report
|
||||
RestoreStorage string // target storage for the restored rootfs (e.g. "local-lvm")
|
||||
ScratchMin int // inclusive scratch VMID band (must be > 0)
|
||||
ScratchMax int // inclusive
|
||||
BootTimeout time.Duration // 0 → DefaultBootTimeout
|
||||
}
|
||||
|
||||
// RestoreTestResult is the reconcile-local outcome (the backup package maps it to the
|
||||
// hub.RestoreTest wire record — reconcile must not import hub for this).
|
||||
type RestoreTestResult struct {
|
||||
Archive string
|
||||
SourceTier string
|
||||
ScratchVMID int
|
||||
Pass bool
|
||||
Verified string // "boot+running" this slice
|
||||
Skipped bool // no free scratch VMID in band → test not run
|
||||
Err error
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch
|
||||
// guest: ClassGuestDestroy made benign by AgentTaggedScratch provenance (classify.go). The
|
||||
// gate authorizes it unsigned but is genuinely in-path (wrong provenance → pending_signature).
|
||||
func IntentForScratchDestroy(hostID string, vmid int) Intent {
|
||||
return Intent{
|
||||
Class: ClassGuestDestroy,
|
||||
HostID: hostID,
|
||||
GuestID: strconv.Itoa(vmid),
|
||||
VMID: vmid,
|
||||
Provenance: Provenance{AgentTaggedScratch: true}, // agent-internal, never hub-sourced
|
||||
Source: SourceOneShotJob,
|
||||
}
|
||||
}
|
||||
|
||||
// RunRestoreTest runs one restore-test on the per-guest queue lane of a fresh scratch VMID.
|
||||
// It journals a Scratch-owned entry BEFORE any mutation, so a crash anywhere after this
|
||||
// point is recoverable (Recover destroys the scratch guest). Teardown runs on EVERY path
|
||||
// (defer), including a failed verify. The returned Err is the TEST verdict's error (restore
|
||||
// or boot failure), independent of teardown success.
|
||||
func (e *Engine) RunRestoreTest(ctx context.Context, spec RestoreTestSpec) RestoreTestResult {
|
||||
now := time.Now().UTC()
|
||||
res := RestoreTestResult{Archive: spec.Archive, SourceTier: spec.SourceTier, StartedAt: now}
|
||||
|
||||
if spec.Archive == "" || spec.RestoreStorage == "" {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test needs an archive and a restore storage")
|
||||
return res
|
||||
}
|
||||
if spec.ScratchMin <= 0 || spec.ScratchMax < spec.ScratchMin {
|
||||
res.Err = fmt.Errorf("reconcile: invalid scratch VMID band [%d,%d]", spec.ScratchMin, spec.ScratchMax)
|
||||
return res
|
||||
}
|
||||
|
||||
lxc, err := e.api.ListLXC(ctx)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test list guests: %w", err)
|
||||
return res
|
||||
}
|
||||
vmid, ok := pickScratchVMID(lxc, spec.ScratchMin, spec.ScratchMax)
|
||||
if !ok {
|
||||
// Full band (e.g. an accumulation of un-torn-down scratch guests) → skip, never
|
||||
// panic or pick out-of-band. Recover will reap any genuinely leaked ones.
|
||||
e.logger.Warn("restore-test skipped: no free scratch VMID in band",
|
||||
"min", spec.ScratchMin, "max", spec.ScratchMax)
|
||||
res.Skipped = true
|
||||
return res
|
||||
}
|
||||
res.ScratchVMID = vmid
|
||||
|
||||
// Serialize on the scratch VMID's lane (inherits §10), and capture the result.
|
||||
ch := e.queue.Submit(vmid, func() error {
|
||||
e.runScratchTest(ctx, vmid, spec, &res)
|
||||
return res.Err
|
||||
})
|
||||
<-ch
|
||||
res.Duration = time.Since(now)
|
||||
return res
|
||||
}
|
||||
|
||||
// runScratchTest is the journaled body (runs on vmid's queue lane).
|
||||
func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestSpec, res *RestoreTestResult) {
|
||||
base := JournalEntry{OpID: e.scratchOpID(vmid), VMID: vmid, Kind: scratchKind, Scratch: true}
|
||||
|
||||
// OWN the scratch guest's cleanup BEFORE any mutation. From here, a crash is recoverable.
|
||||
e.append(withState(base, OpStarted))
|
||||
|
||||
// Teardown ALWAYS runs (even on a failed verify). Uses a cancel-immune context so a
|
||||
// daemon shutdown mid-test still tears down; if teardown fails, the entry stays
|
||||
// in-flight and Recover reaps the guest on the next start.
|
||||
defer e.teardownScratch(ctx, base)
|
||||
|
||||
// 1. Restore into the fresh scratch VMID (benign create path). The UPID is for error
|
||||
// detection only — it does NOT make the Scratch entry terminal (teardown does).
|
||||
upid, err := e.api.RestoreLXC(ctx, proxmox.RestoreLXCOptions{
|
||||
VMID: vmid, Archive: spec.Archive, Storage: spec.RestoreStorage,
|
||||
})
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test restore: %w", err)
|
||||
return
|
||||
}
|
||||
e.append(withUPID(base, upid, OpTaskRunning))
|
||||
if upid != "" {
|
||||
if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test restore task: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Net link-down on every interface BEFORE boot — test-safety so the clone (which
|
||||
// keeps the source MAC/hostname; identity-reset is slice 7) can't conflict with a
|
||||
// running source on L2/IP. Benign SetConfig.
|
||||
cfg, err := e.api.GuestConfig(ctx, vmid)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test read scratch config: %w", err)
|
||||
return
|
||||
}
|
||||
for key, val := range cfg.Nets() {
|
||||
if _, err := e.api.SetConfig(ctx, vmid, map[string]string{key: withLinkDown(val)}); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test net link-down %s: %w", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8).
|
||||
startUPID, err := e.api.Start(ctx, vmid)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test start: %w", err)
|
||||
return
|
||||
}
|
||||
if startUPID != "" {
|
||||
if _, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{}); err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil {
|
||||
res.Err = err
|
||||
return
|
||||
}
|
||||
res.Pass = true
|
||||
res.Verified = "boot+running"
|
||||
}
|
||||
|
||||
// teardownScratch destroys the scratch guest (benign, gated) and records the entry terminal.
|
||||
// On any teardown failure it leaves the entry in-flight so Recover reaps the guest later.
|
||||
func (e *Engine) teardownScratch(ctx context.Context, base JournalEntry) {
|
||||
// Cancel-immune + bounded, so a shutdown mid-test still tears down.
|
||||
tctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, base.VMID), nil)
|
||||
if !dec.Allowed {
|
||||
e.logger.Error("restore-test: scratch teardown refused by gate (unexpected); left for Recover",
|
||||
"vmid", base.VMID, "reason", dec.Reason)
|
||||
return
|
||||
}
|
||||
upid, err := e.api.DestroyLXC(tctx, base.VMID)
|
||||
if err != nil {
|
||||
e.logger.Error("restore-test: scratch teardown failed; left for Recover", "vmid", base.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := e.api.WaitTask(tctx, upid, proxmox.WaitOptions{}); err != nil {
|
||||
e.logger.Error("restore-test: scratch teardown task failed; left for Recover", "vmid", base.VMID, "err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
e.append(withState(base, OpSucceeded))
|
||||
e.logger.Info("restore-test: scratch guest torn down", "vmid", base.VMID)
|
||||
}
|
||||
|
||||
// waitRunning polls GuestStatus until the guest is running or the timeout elapses. The poll
|
||||
// interval is 2s in production, but shrinks for short timeouts so it stays responsive.
|
||||
func (e *Engine) waitRunning(ctx context.Context, vmid int, timeout time.Duration) error {
|
||||
interval := 2 * time.Second
|
||||
if timeout < 4*interval {
|
||||
if interval = timeout / 4; interval < 10*time.Millisecond {
|
||||
interval = 10 * time.Millisecond
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
g, err := e.api.GuestStatus(ctx, vmid)
|
||||
if err == nil && g.Status == "running" {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s (last err: %w)", vmid, timeout, err)
|
||||
}
|
||||
return fmt.Errorf("reconcile: restore-test verify: guest %d not running within %s", vmid, timeout)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pickScratchVMID returns the lowest free VMID in [min,max], excluding the standing 9999
|
||||
// scratch and any in-use guest. ok=false when the band is fully occupied (the test is then
|
||||
// skipped, never run out-of-band).
|
||||
func pickScratchVMID(lxc []proxmox.Guest, min, max int) (int, bool) {
|
||||
used := make(map[int]bool, len(lxc))
|
||||
for _, g := range lxc {
|
||||
used[g.VMID] = true
|
||||
}
|
||||
for id := min; id <= max; id++ {
|
||||
if id == 9999 || used[id] {
|
||||
continue
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// withLinkDown sets link_down=1 on a Proxmox netN config string, REPLACING any existing
|
||||
// link_down token (never blind-concatenating, so a re-applied/pre-set value can't produce a
|
||||
// malformed netN).
|
||||
func withLinkDown(netN string) string {
|
||||
parts := strings.Split(netN, ",")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p == "" || strings.HasPrefix(p, "link_down=") {
|
||||
continue
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
out = append(out, "link_down=1")
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func bootTimeout(spec RestoreTestSpec) time.Duration {
|
||||
if spec.BootTimeout > 0 {
|
||||
return spec.BootTimeout
|
||||
}
|
||||
return DefaultBootTimeout
|
||||
}
|
||||
|
||||
func (e *Engine) scratchOpID(vmid int) string {
|
||||
return "scratch-restore-" + strconv.Itoa(vmid) + "-" + nextSeq(&e.opSeq)
|
||||
}
|
||||
|
||||
// withState / withUPID build journal records from a base entry, preserving its identity +
|
||||
// Scratch flag.
|
||||
func withState(base JournalEntry, state OpState) JournalEntry {
|
||||
base.State = state
|
||||
base.At = time.Now().UTC()
|
||||
return base
|
||||
}
|
||||
func withUPID(base JournalEntry, upid string, state OpState) JournalEntry {
|
||||
base.UPID = upid
|
||||
base.State = state
|
||||
base.At = time.Now().UTC()
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// scratchCfg builds a fake GuestConfig with one net interface (so the link-down SetConfig
|
||||
// step runs).
|
||||
func scratchCfg() proxmox.GuestConfig {
|
||||
return proxmox.GuestConfig{Extra: map[string]json.RawMessage{
|
||||
"net0": json.RawMessage(`"name=eth0,bridge=vmbr0,hwaddr=AA:BB:CC:DD:EE:FF,ip=dhcp"`),
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRunRestoreTest_PassAndTeardown(t *testing.T) {
|
||||
api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()}} // empty lxc → 990000 free; running default
|
||||
e, j, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "local:backup/x.tar.zst", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, SourceTier: "local",
|
||||
})
|
||||
if res.Skipped || !res.Pass || res.Err != nil {
|
||||
t.Fatalf("expected pass, got %+v", res)
|
||||
}
|
||||
if res.ScratchVMID != 990000 || res.Verified != "boot+running" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if len(api.restores) != 1 || api.restores[0].VMID != 990000 || api.restores[0].Archive != "local:backup/x.tar.zst" {
|
||||
t.Fatalf("restore not issued correctly: %+v", api.restores)
|
||||
}
|
||||
// net link-down applied before boot.
|
||||
foundLinkDown := false
|
||||
for _, s := range api.sets {
|
||||
if s.vmid == 990000 && s.params["net0"] != "" && contains2(s.params["net0"], "link_down=1") {
|
||||
foundLinkDown = true
|
||||
}
|
||||
}
|
||||
if !foundLinkDown {
|
||||
t.Errorf("expected a net link-down SetConfig, got %+v", api.sets)
|
||||
}
|
||||
// teardown destroyed the scratch guest, and the journal entry is terminal (not in-flight).
|
||||
if len(api.destroys) != 1 || api.destroys[0] != 990000 {
|
||||
t.Fatalf("scratch not torn down: %+v", api.destroys)
|
||||
}
|
||||
if len(j.InFlight()) != 0 {
|
||||
t.Errorf("scratch entry must be terminal after teardown: %+v", j.InFlight())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRestoreTest_TeardownOnFailedVerify(t *testing.T) {
|
||||
// Guest never reaches running → verify fails, but teardown MUST still run.
|
||||
api := &fakeAPI{
|
||||
cfg: map[int]proxmox.GuestConfig{990000: scratchCfg()},
|
||||
status: map[int]proxmox.Guest{990000: {VMID: 990000, Status: "stopped"}},
|
||||
}
|
||||
e, j, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "vol", RestoreStorage: "local-lvm",
|
||||
ScratchMin: 990000, ScratchMax: 990009, BootTimeout: 40 * time.Millisecond,
|
||||
})
|
||||
if res.Pass || res.Err == nil {
|
||||
t.Fatalf("expected a failed verify, got %+v", res)
|
||||
}
|
||||
if len(api.destroys) != 1 || api.destroys[0] != 990000 {
|
||||
t.Fatalf("teardown MUST run even on a failed verify: destroys=%+v", api.destroys)
|
||||
}
|
||||
if len(j.InFlight()) != 0 {
|
||||
t.Errorf("scratch entry must be terminal after teardown: %+v", j.InFlight())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRestoreTest_RestoreFailureStillTearsDown(t *testing.T) {
|
||||
api := &fakeAPI{restoreErr: errors.New("restore boom")}
|
||||
e, j, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
|
||||
})
|
||||
if res.Pass || res.Err == nil {
|
||||
t.Fatalf("expected restore failure, got %+v", res)
|
||||
}
|
||||
// Even though restore failed, the scratch entry was journaled BEFORE the restore, so
|
||||
// teardown runs (idempotent — destroys the maybe-partial guest).
|
||||
if len(api.destroys) != 1 {
|
||||
t.Fatalf("teardown must run after a restore failure: %+v", api.destroys)
|
||||
}
|
||||
if len(j.InFlight()) != 0 {
|
||||
t.Errorf("scratch entry must be terminal: %+v", j.InFlight())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRestoreTest_FullBandSkips(t *testing.T) {
|
||||
// Whole band occupied → skipped, never run / out-of-band.
|
||||
var guests []proxmox.Guest
|
||||
for id := 990000; id <= 990001; id++ {
|
||||
guests = append(guests, proxmox.Guest{VMID: id})
|
||||
}
|
||||
api := &fakeAPI{lxc: guests}
|
||||
e, _, q := newEngine(t, api, EmptyProvider{})
|
||||
defer q.Close()
|
||||
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
|
||||
Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990001,
|
||||
})
|
||||
if !res.Skipped {
|
||||
t.Fatalf("full band must skip, got %+v", res)
|
||||
}
|
||||
if len(api.restores) != 0 || len(api.destroys) != 0 {
|
||||
t.Errorf("a skipped test must not restore or destroy anything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRestoreTest_InvalidBandErrors(t *testing.T) {
|
||||
e, _, q := newEngine(t, &fakeAPI{}, EmptyProvider{})
|
||||
defer q.Close()
|
||||
res := e.RunRestoreTest(context.Background(), RestoreTestSpec{Archive: "v", RestoreStorage: "s", ScratchMin: 0})
|
||||
if res.Err == nil {
|
||||
t.Fatal("an invalid scratch band must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickScratchVMID(t *testing.T) {
|
||||
// excludes 9999 and in-use; lowest free.
|
||||
got, ok := pickScratchVMID([]proxmox.Guest{{VMID: 990000}}, 990000, 990009)
|
||||
if !ok || got != 990001 {
|
||||
t.Errorf("pick = %d,%v want 990001,true", got, ok)
|
||||
}
|
||||
// full band.
|
||||
full := []proxmox.Guest{{VMID: 990000}, {VMID: 990001}}
|
||||
if _, ok := pickScratchVMID(full, 990000, 990001); ok {
|
||||
t.Error("full band must return ok=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithLinkDown(t *testing.T) {
|
||||
got := withLinkDown("name=eth0,bridge=vmbr0,ip=dhcp")
|
||||
if !contains2(got, "link_down=1") || !contains2(got, "name=eth0") {
|
||||
t.Errorf("withLinkDown lost fields or didn't set link_down: %q", got)
|
||||
}
|
||||
// idempotent: an existing link_down is replaced, not duplicated.
|
||||
got = withLinkDown("name=eth0,link_down=0,bridge=vmbr0")
|
||||
if count(got, "link_down=") != 1 || !contains2(got, "link_down=1") {
|
||||
t.Errorf("withLinkDown must replace an existing link_down (got %q)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- recover the leaked scratch guest (the headline crash-safety test) ---
|
||||
|
||||
func TestRecover_LeakedScratchDestroyed(t *testing.T) {
|
||||
// The scratch guest still exists at startup (agent crashed mid-test) → Recover destroys it.
|
||||
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 990000, Status: "running"}}}
|
||||
e, j, _ := newEngine(t, api, EmptyProvider{})
|
||||
if err := j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := e.Recover(context.Background())
|
||||
if res.ScratchDestroyed != 1 {
|
||||
t.Fatalf("leaked scratch must be destroyed, got %+v", res)
|
||||
}
|
||||
if len(api.destroys) != 1 || api.destroys[0] != 990000 {
|
||||
t.Fatalf("DestroyLXC not called for the leaked scratch: %+v", api.destroys)
|
||||
}
|
||||
if len(j.InFlight()) != 0 {
|
||||
t.Errorf("resolved scratch entry must not be in-flight: %+v", j.InFlight())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecover_LeakedScratchAlreadyGone(t *testing.T) {
|
||||
// Crash AFTER the destroy task but BEFORE the terminal record → guest already gone →
|
||||
// idempotent clean (no destroy issued).
|
||||
api := &fakeAPI{lxc: []proxmox.Guest{{VMID: 9001, Status: "stopped"}}} // 990000 absent
|
||||
e, j, _ := newEngine(t, api, EmptyProvider{})
|
||||
j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()})
|
||||
res := e.Recover(context.Background())
|
||||
if res.ScratchClean != 1 || len(api.destroys) != 0 {
|
||||
t.Fatalf("already-gone scratch must be clean with no destroy, got res=%+v destroys=%+v", res, api.destroys)
|
||||
}
|
||||
if len(j.InFlight()) != 0 {
|
||||
t.Errorf("entry must be resolved: %+v", j.InFlight())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecover_LeakedScratchListUnreadable(t *testing.T) {
|
||||
api := &fakeAPI{listErr: errors.New("api down")}
|
||||
e, j, _ := newEngine(t, api, EmptyProvider{})
|
||||
j.Append(JournalEntry{OpID: "scratch-990000-1", VMID: 990000, Kind: scratchKind, Scratch: true, State: OpTaskRunning, At: time.Now().UTC()})
|
||||
res := e.Recover(context.Background())
|
||||
if res.Unresolved != 1 || len(j.InFlight()) != 1 {
|
||||
t.Fatalf("unreadable list must leave the scratch in-flight for a later Recover, got res=%+v inflight=%d", res, len(j.InFlight()))
|
||||
}
|
||||
if len(api.destroys) != 0 {
|
||||
t.Error("must not destroy when it can't confirm the guest exists")
|
||||
}
|
||||
}
|
||||
|
||||
// small string helpers (avoid importing strings in the test for one call).
|
||||
func contains2(s, sub string) bool { return indexOf(s, sub) >= 0 }
|
||||
func count(s, sub string) int {
|
||||
n, i := 0, 0
|
||||
for {
|
||||
j := indexOf(s[i:], sub)
|
||||
if j < 0 {
|
||||
return n
|
||||
}
|
||||
n++
|
||||
i += j + len(sub)
|
||||
}
|
||||
}
|
||||
func indexOf(s, sub string) int {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -114,6 +114,14 @@ type GuestAPI interface {
|
||||
SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error)
|
||||
// ResizeLXC grows a volume (grow-only; the planner never emits a shrink). Async → UPID.
|
||||
ResizeLXC(ctx context.Context, vmid int, disk, size string) (string, error)
|
||||
// RestoreLXC restores an archive into a (fresh) vmid — the create path (slice 6). Async → UPID.
|
||||
RestoreLXC(ctx context.Context, opts proxmox.RestoreLXCOptions) (string, error)
|
||||
// DestroyLXC destroys a guest — the scratch-teardown primitive (slice 6). Async → UPID.
|
||||
// Destructive-class; the engine only ever issues it for an agent-tagged scratch guest
|
||||
// (benign by provenance) via the gate.
|
||||
DestroyLXC(ctx context.Context, vmid int) (string, error)
|
||||
// GuestStatus reads a single guest's current status (run-state poll during a restore-test).
|
||||
GuestStatus(ctx context.Context, vmid int) (proxmox.Guest, error)
|
||||
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
||||
// TaskStatusOnce is a single non-blocking task-status read — used by crash
|
||||
// recovery to learn the outcome of an op that was in flight when the agent died.
|
||||
|
||||
Reference in New Issue
Block a user