v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)
The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→ teardown) that closes "a backup you haven't restored isn't a backup". All benign (reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent. - proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID. - reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent). - internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence scheduler (4th daemon goroutine, default 24h) + in-memory report store. - hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical + bidirectional key-set tests; hub handler logs a FAILED restore-test prominently. - config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// BackupAPI is the read+backup proxmox surface the runner needs. *proxmox.Client satisfies it.
|
||||
type BackupAPI interface {
|
||||
Vzdump(ctx context.Context, opts proxmox.VzdumpOptions) (string, error)
|
||||
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
||||
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||||
StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error)
|
||||
}
|
||||
|
||||
// BackupRunner orchestrates a crash-consistent vzdump to a local target and reports the
|
||||
// result (incl. the bulk-volume gap). An agent-initiated vzdump is crash-consistent only
|
||||
// (no fsfreeze); the report says so.
|
||||
type BackupRunner struct {
|
||||
api BackupAPI
|
||||
target string // backup storage (content=backup)
|
||||
mode proxmox.BackupMode // default ModeSnapshot
|
||||
notes string // optional notes-template
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
|
||||
// for lvm-thin); the caller may pass ModeStop for storages without snapshot support.
|
||||
func NewBackupRunner(api BackupAPI, target string, mode proxmox.BackupMode, notes string, logger *slog.Logger) *BackupRunner {
|
||||
if mode == "" {
|
||||
mode = proxmox.ModeSnapshot
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, logger: logger, now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
// Backup runs one vzdump of vmid to the local target and returns the report record. A
|
||||
// failure is returned BOTH as an error and as a Backup{Success:false,...} so the caller can
|
||||
// record the failed attempt.
|
||||
func (r *BackupRunner) Backup(ctx context.Context, vmid int) (hub.Backup, error) {
|
||||
start := r.now()
|
||||
rec := hub.Backup{
|
||||
TargetID: r.target,
|
||||
VMID: vmid,
|
||||
Mode: string(r.mode),
|
||||
CrashConsistent: true, // always, this slice (no controller quiesce)
|
||||
StartedAt: start.Format(time.RFC3339),
|
||||
}
|
||||
if r.target == "" {
|
||||
rec.Error = "no backup target configured"
|
||||
return rec, fmt.Errorf("backup: %s", rec.Error)
|
||||
}
|
||||
|
||||
// Bulk-volume gap: which mountpoints the vzdump will EXCLUDE (best-effort; a config-read
|
||||
// failure just leaves the gap unknown, never fails the backup).
|
||||
if cfg, err := r.api.GuestConfig(ctx, vmid); err == nil {
|
||||
rec.UncoveredVolumes = uncoveredMountpoints(cfg.MountPoints())
|
||||
} else {
|
||||
r.logger.Warn("backup: could not read guest config for bulk-gap", "vmid", vmid, "err", err)
|
||||
rec.UncoveredVolumes = []string{}
|
||||
}
|
||||
|
||||
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
||||
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
||||
})
|
||||
if err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: 30 * time.Minute}); err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the produced archive (volid + size) — the task status carries no result volid.
|
||||
vol, size, err := r.latestArchive(ctx, vmid)
|
||||
if err != nil {
|
||||
rec.Error = fmt.Sprintf("backup succeeded but archive not resolved: %v", err)
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: resolve archive vmid %d: %w", vmid, err)
|
||||
}
|
||||
rec.Archive = vol
|
||||
rec.SizeBytes = size
|
||||
rec.Success = true
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
r.logger.Info("backup: completed", "vmid", vmid, "target", r.target, "archive", vol, "size_bytes", size,
|
||||
"uncovered_volumes", len(rec.UncoveredVolumes))
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// PickRestoreCandidate returns the newest backup archive on the target (any guest), or ""
|
||||
// when there is none — the restore-test then no-ops cleanly.
|
||||
func (r *BackupRunner) PickRestoreCandidate(ctx context.Context) (string, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var best string
|
||||
var bestCTime int64 = -1
|
||||
for _, e := range contents {
|
||||
if e.Content == "backup" && e.CTime > bestCTime {
|
||||
bestCTime, best = e.CTime, e.VolID
|
||||
}
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
// latestArchive finds the newest backup archive volid + size for vmid on the target.
|
||||
func (r *BackupRunner) latestArchive(ctx context.Context, vmid int) (string, int64, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var vol string
|
||||
var size, bestCTime int64 = 0, -1
|
||||
for _, e := range contents {
|
||||
if e.Content == "backup" && e.VMID == vmid && e.CTime > bestCTime {
|
||||
bestCTime, vol, size = e.CTime, e.VolID, e.Size
|
||||
}
|
||||
}
|
||||
if vol == "" {
|
||||
return "", 0, fmt.Errorf("no backup archive found for vmid %d on %s", vmid, r.target)
|
||||
}
|
||||
return vol, size, nil
|
||||
}
|
||||
|
||||
// uncoveredMountpoints returns the mountpoint paths the guest vzdump EXCLUDES. LXC mount
|
||||
// points are OPT-IN to vzdump: a mpN with `backup=1` is covered; ANY other state — the
|
||||
// `backup=` token absent OR `backup=0` — is excluded. We deliberately treat unset as
|
||||
// uncovered (the safe DR direction: never imply an unprotected volume is backed up).
|
||||
func uncoveredMountpoints(mps map[string]string) []string {
|
||||
var out []string
|
||||
for key, cfg := range mps {
|
||||
if mountpointCovered(cfg) {
|
||||
continue
|
||||
}
|
||||
out = append(out, mountpointLabel(key, cfg))
|
||||
}
|
||||
sort.Strings(out)
|
||||
if out == nil {
|
||||
return []string{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mountpointCovered reports whether a mpN config string is included in the vzdump — true
|
||||
// ONLY when it carries an explicit `backup=1`.
|
||||
func mountpointCovered(cfg string) bool {
|
||||
for _, tok := range strings.Split(cfg, ",") {
|
||||
if v, ok := strings.CutPrefix(tok, "backup="); ok {
|
||||
return v == "1"
|
||||
}
|
||||
}
|
||||
return false // no backup= token → opt-out by default → not covered
|
||||
}
|
||||
|
||||
// mountpointLabel prefers the mp=/path token, falling back to the mpN key.
|
||||
func mountpointLabel(key, cfg string) string {
|
||||
for _, tok := range strings.Split(cfg, ",") {
|
||||
if p, ok := strings.CutPrefix(tok, "mp="); ok && p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// ToHubRestoreTest maps a reconcile restore-test result to the hub wire record (the backup
|
||||
// package owns the reconcile→hub mapping so reconcile need not import hub for the result).
|
||||
func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.RestoreTest {
|
||||
rt := hub.RestoreTest{
|
||||
SourceArchive: res.Archive,
|
||||
SourceTier: res.SourceTier,
|
||||
ScratchVMID: res.ScratchVMID,
|
||||
Pass: res.Pass,
|
||||
Verified: res.Verified,
|
||||
TestedAt: testedAt.Format(time.RFC3339),
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
}
|
||||
if res.Err != nil {
|
||||
rt.Error = res.Err.Error()
|
||||
}
|
||||
return rt
|
||||
}
|
||||
Reference in New Issue
Block a user