Files
felhom-agent/internal/backup/runner.go
T
admin b527430ec7 v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)
The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→
teardown) that closes "a backup you haven't restored isn't a backup". All benign
(reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local
target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent.

- proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID.
- reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down
  pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a
  leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent).
- internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence
  scheduler (4th daemon goroutine, default 24h) + in-memory report store.
- hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical +
  bidirectional key-set tests; hub handler logs a FAILED restore-test prominently.
- config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 13:49:39 +02:00

199 lines
6.9 KiB
Go

package backup
import (
"context"
"fmt"
"log/slog"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
)
// BackupAPI is the read+backup proxmox surface the runner needs. *proxmox.Client satisfies it.
type BackupAPI interface {
Vzdump(ctx context.Context, opts proxmox.VzdumpOptions) (string, error)
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error)
}
// BackupRunner orchestrates a crash-consistent vzdump to a local target and reports the
// result (incl. the bulk-volume gap). An agent-initiated vzdump is crash-consistent only
// (no fsfreeze); the report says so.
type BackupRunner struct {
api BackupAPI
target string // backup storage (content=backup)
mode proxmox.BackupMode // default ModeSnapshot
notes string // optional notes-template
logger *slog.Logger
now func() time.Time
}
// NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
// for lvm-thin); the caller may pass ModeStop for storages without snapshot support.
func NewBackupRunner(api BackupAPI, target string, mode proxmox.BackupMode, notes string, logger *slog.Logger) *BackupRunner {
if mode == "" {
mode = proxmox.ModeSnapshot
}
if logger == nil {
logger = slog.Default()
}
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, logger: logger, now: func() time.Time { return time.Now().UTC() }}
}
// Backup runs one vzdump of vmid to the local target and returns the report record. A
// failure is returned BOTH as an error and as a Backup{Success:false,...} so the caller can
// record the failed attempt.
func (r *BackupRunner) Backup(ctx context.Context, vmid int) (hub.Backup, error) {
start := r.now()
rec := hub.Backup{
TargetID: r.target,
VMID: vmid,
Mode: string(r.mode),
CrashConsistent: true, // always, this slice (no controller quiesce)
StartedAt: start.Format(time.RFC3339),
}
if r.target == "" {
rec.Error = "no backup target configured"
return rec, fmt.Errorf("backup: %s", rec.Error)
}
// Bulk-volume gap: which mountpoints the vzdump will EXCLUDE (best-effort; a config-read
// failure just leaves the gap unknown, never fails the backup).
if cfg, err := r.api.GuestConfig(ctx, vmid); err == nil {
rec.UncoveredVolumes = uncoveredMountpoints(cfg.MountPoints())
} else {
r.logger.Warn("backup: could not read guest config for bulk-gap", "vmid", vmid, "err", err)
rec.UncoveredVolumes = []string{}
}
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
})
if err != nil {
rec.Error = err.Error()
rec.DurationSeconds = time.Since(start).Seconds()
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
}
if upid != "" {
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: 30 * time.Minute}); err != nil {
rec.Error = err.Error()
rec.DurationSeconds = time.Since(start).Seconds()
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
}
}
// Resolve the produced archive (volid + size) — the task status carries no result volid.
vol, size, err := r.latestArchive(ctx, vmid)
if err != nil {
rec.Error = fmt.Sprintf("backup succeeded but archive not resolved: %v", err)
rec.DurationSeconds = time.Since(start).Seconds()
return rec, fmt.Errorf("backup: resolve archive vmid %d: %w", vmid, err)
}
rec.Archive = vol
rec.SizeBytes = size
rec.Success = true
rec.DurationSeconds = time.Since(start).Seconds()
r.logger.Info("backup: completed", "vmid", vmid, "target", r.target, "archive", vol, "size_bytes", size,
"uncovered_volumes", len(rec.UncoveredVolumes))
return rec, nil
}
// PickRestoreCandidate returns the newest backup archive on the target (any guest), or ""
// when there is none — the restore-test then no-ops cleanly.
func (r *BackupRunner) PickRestoreCandidate(ctx context.Context) (string, error) {
contents, err := r.api.StorageContent(ctx, r.target)
if err != nil {
return "", err
}
var best string
var bestCTime int64 = -1
for _, e := range contents {
if e.Content == "backup" && e.CTime > bestCTime {
bestCTime, best = e.CTime, e.VolID
}
}
return best, nil
}
// latestArchive finds the newest backup archive volid + size for vmid on the target.
func (r *BackupRunner) latestArchive(ctx context.Context, vmid int) (string, int64, error) {
contents, err := r.api.StorageContent(ctx, r.target)
if err != nil {
return "", 0, err
}
var vol string
var size, bestCTime int64 = 0, -1
for _, e := range contents {
if e.Content == "backup" && e.VMID == vmid && e.CTime > bestCTime {
bestCTime, vol, size = e.CTime, e.VolID, e.Size
}
}
if vol == "" {
return "", 0, fmt.Errorf("no backup archive found for vmid %d on %s", vmid, r.target)
}
return vol, size, nil
}
// uncoveredMountpoints returns the mountpoint paths the guest vzdump EXCLUDES. LXC mount
// points are OPT-IN to vzdump: a mpN with `backup=1` is covered; ANY other state — the
// `backup=` token absent OR `backup=0` — is excluded. We deliberately treat unset as
// uncovered (the safe DR direction: never imply an unprotected volume is backed up).
func uncoveredMountpoints(mps map[string]string) []string {
var out []string
for key, cfg := range mps {
if mountpointCovered(cfg) {
continue
}
out = append(out, mountpointLabel(key, cfg))
}
sort.Strings(out)
if out == nil {
return []string{}
}
return out
}
// mountpointCovered reports whether a mpN config string is included in the vzdump — true
// ONLY when it carries an explicit `backup=1`.
func mountpointCovered(cfg string) bool {
for _, tok := range strings.Split(cfg, ",") {
if v, ok := strings.CutPrefix(tok, "backup="); ok {
return v == "1"
}
}
return false // no backup= token → opt-out by default → not covered
}
// mountpointLabel prefers the mp=/path token, falling back to the mpN key.
func mountpointLabel(key, cfg string) string {
for _, tok := range strings.Split(cfg, ",") {
if p, ok := strings.CutPrefix(tok, "mp="); ok && p != "" {
return p
}
}
return key
}
// ToHubRestoreTest maps a reconcile restore-test result to the hub wire record (the backup
// package owns the reconcile→hub mapping so reconcile need not import hub for the result).
func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.RestoreTest {
rt := hub.RestoreTest{
SourceArchive: res.Archive,
SourceTier: res.SourceTier,
ScratchVMID: res.ScratchVMID,
Pass: res.Pass,
Verified: res.Verified,
TestedAt: testedAt.Format(time.RFC3339),
DurationSeconds: res.Duration.Seconds(),
}
if res.Err != nil {
rt.Error = res.Err.Error()
}
return rt
}