v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)
The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→ teardown) that closes "a backup you haven't restored isn't a backup". All benign (reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent. - proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID. - reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent). - internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence scheduler (4th daemon goroutine, default 24h) + in-memory report store. - hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical + bidirectional key-set tests; hub handler logs a FAILED restore-test prominently. - config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||
|
||||
// fakeBackupAPI is a synthetic BackupAPI.
|
||||
type fakeBackupAPI struct {
|
||||
vzdumpUPID string
|
||||
vzdumpErr error
|
||||
waitErr error
|
||||
cfg proxmox.GuestConfig
|
||||
cfgErr error
|
||||
content []proxmox.StorageContent
|
||||
contentErr error
|
||||
vzdumps []proxmox.VzdumpOptions
|
||||
}
|
||||
|
||||
func (f *fakeBackupAPI) Vzdump(_ context.Context, o proxmox.VzdumpOptions) (string, error) {
|
||||
f.vzdumps = append(f.vzdumps, o)
|
||||
return f.vzdumpUPID, f.vzdumpErr
|
||||
}
|
||||
func (f *fakeBackupAPI) WaitTask(_ context.Context, _ string, _ proxmox.WaitOptions) (proxmox.TaskStatus, error) {
|
||||
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, f.waitErr
|
||||
}
|
||||
func (f *fakeBackupAPI) GuestConfig(_ context.Context, _ int) (proxmox.GuestConfig, error) {
|
||||
return f.cfg, f.cfgErr
|
||||
}
|
||||
func (f *fakeBackupAPI) StorageContent(_ context.Context, _ string) ([]proxmox.StorageContent, error) {
|
||||
return f.content, f.contentErr
|
||||
}
|
||||
|
||||
// guestCfgWithMounts builds a GuestConfig whose Extra carries the given mpN strings.
|
||||
func guestCfgWithMounts(mps map[string]string) proxmox.GuestConfig {
|
||||
extra := map[string]json.RawMessage{}
|
||||
for k, v := range mps {
|
||||
b, _ := json.Marshal(v)
|
||||
extra[k] = b
|
||||
}
|
||||
return proxmox.GuestConfig{Extra: extra}
|
||||
}
|
||||
|
||||
func TestBackup_SuccessResolvesArchiveAndBulkGap(t *testing.T) {
|
||||
api := &fakeBackupAPI{
|
||||
vzdumpUPID: "UPID:vzdump:1",
|
||||
cfg: guestCfgWithMounts(map[string]string{
|
||||
"mp0": "local-lvm:8,mp=/mnt/bulk,backup=0", // explicit opt-out → uncovered
|
||||
"mp1": "local-lvm:4,mp=/mnt/db,backup=1", // covered
|
||||
"mp2": "local-lvm:2,mp=/mnt/scratch", // UNSET → uncovered (opt-in default)
|
||||
}),
|
||||
content: []proxmox.StorageContent{
|
||||
{VolID: "local:backup/old-9001.tar.zst", Content: "backup", VMID: 9001, Size: 100, CTime: 100},
|
||||
{VolID: "local:backup/new-9001.tar.zst", Content: "backup", VMID: 9001, Size: 524288000, CTime: 200},
|
||||
{VolID: "local:backup/other-9002.tar.zst", Content: "backup", VMID: 9002, Size: 7, CTime: 999},
|
||||
},
|
||||
}
|
||||
r := NewBackupRunner(api, "local", "", "felhom test", quiet())
|
||||
rec, err := r.Backup(context.Background(), 9001)
|
||||
if err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
if !rec.Success || !rec.CrashConsistent {
|
||||
t.Errorf("record = %+v, want success + crash_consistent", rec)
|
||||
}
|
||||
if rec.Archive != "local:backup/new-9001.tar.zst" || rec.SizeBytes != 524288000 {
|
||||
t.Errorf("resolved wrong archive/size: %+v", rec)
|
||||
}
|
||||
if rec.Mode != string(proxmox.ModeSnapshot) {
|
||||
t.Errorf("mode = %q, want snapshot (default)", rec.Mode)
|
||||
}
|
||||
// Bulk gap: mp0 (backup=0) AND mp2 (unset) are uncovered; mp1 (backup=1) is NOT.
|
||||
if got := rec.UncoveredVolumes; len(got) != 2 || !has(got, "/mnt/bulk") || !has(got, "/mnt/scratch") {
|
||||
t.Errorf("uncovered = %v, want [/mnt/bulk /mnt/scratch] (unset is uncovered too)", got)
|
||||
}
|
||||
if has(rec.UncoveredVolumes, "/mnt/db") {
|
||||
t.Error("backup=1 mountpoint must NOT be reported uncovered")
|
||||
}
|
||||
// Notes-template threaded through.
|
||||
if len(api.vzdumps) != 1 || api.vzdumps[0].Notes != "felhom test" {
|
||||
t.Errorf("vzdump opts = %+v", api.vzdumps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackup_VzdumpFailureReturnsFailedRecord(t *testing.T) {
|
||||
api := &fakeBackupAPI{vzdumpErr: errors.New("vzdump boom")}
|
||||
r := NewBackupRunner(api, "local", "", "", quiet())
|
||||
rec, err := r.Backup(context.Background(), 9001)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if rec.Success || rec.Error == "" {
|
||||
t.Errorf("failed backup must produce a Success=false record with an Error: %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickRestoreCandidate_NewestOrEmpty(t *testing.T) {
|
||||
api := &fakeBackupAPI{content: []proxmox.StorageContent{
|
||||
{VolID: "a", Content: "backup", CTime: 10},
|
||||
{VolID: "b", Content: "backup", CTime: 99},
|
||||
{VolID: "iso", Content: "iso", CTime: 999}, // not a backup → ignored
|
||||
}}
|
||||
r := NewBackupRunner(api, "local", "", "", quiet())
|
||||
vol, err := r.PickRestoreCandidate(context.Background())
|
||||
if err != nil || vol != "b" {
|
||||
t.Fatalf("pick = %q,%v want newest 'b'", vol, err)
|
||||
}
|
||||
// no backups → "".
|
||||
api.content = []proxmox.StorageContent{{VolID: "iso", Content: "iso"}}
|
||||
if vol, _ := r.PickRestoreCandidate(context.Background()); vol != "" {
|
||||
t.Errorf("no backup → empty, got %q", vol)
|
||||
}
|
||||
}
|
||||
|
||||
// --- scheduler ---
|
||||
|
||||
type fakeRTRunner struct {
|
||||
res reconcile.RestoreTestResult
|
||||
runs int
|
||||
}
|
||||
|
||||
func (f *fakeRTRunner) RunRestoreTest(_ context.Context, _ reconcile.RestoreTestSpec) reconcile.RestoreTestResult {
|
||||
f.runs++
|
||||
return f.res
|
||||
}
|
||||
|
||||
func TestScheduler_TickRunsAndRecords(t *testing.T) {
|
||||
store := NewStore()
|
||||
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Archive: "vol", Pass: true, Verified: "boot+running", Duration: time.Second}}
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) { return "vol", nil },
|
||||
Store: store,
|
||||
Cadence: time.Hour,
|
||||
Logger: quiet(),
|
||||
})
|
||||
s.tick(context.Background())
|
||||
if rt.runs != 1 {
|
||||
t.Fatalf("tick should run the restore-test once, got %d", rt.runs)
|
||||
}
|
||||
got := store.RestoreTests(context.Background())
|
||||
if len(got) != 1 || !got[0].Pass || got[0].SourceArchive != "vol" {
|
||||
t.Fatalf("store should have the recorded result: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_TickNoBackupNoOp(t *testing.T) {
|
||||
rt := &fakeRTRunner{}
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt, Pick: func(context.Context) (string, error) { return "", nil },
|
||||
Store: NewStore(), Cadence: time.Hour, Logger: quiet(),
|
||||
})
|
||||
s.tick(context.Background())
|
||||
if rt.runs != 0 {
|
||||
t.Errorf("no backup available → no restore-test run, got %d", rt.runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_DisabledRunReturnsOnCancel(t *testing.T) {
|
||||
s := NewScheduler(SchedulerOptions{Cadence: 0, Logger: quiet()})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Run(ctx) }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("disabled scheduler Run should return nil on cancel, got %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("disabled scheduler did not return on cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_Reporters(t *testing.T) {
|
||||
s := NewStore()
|
||||
if len(s.Backups(context.Background())) != 0 || len(s.RestoreTests(context.Background())) != 0 {
|
||||
t.Fatal("empty store must report empty (non-nil) slices")
|
||||
}
|
||||
}
|
||||
|
||||
func has(ss []string, want string) bool {
|
||||
for _, s := range ss {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Package backup is the guest-level backup + self-restore-test layer (doc 03 §8, slice 6
|
||||
// Phase A). It orchestrates a crash-consistent vzdump to a LOCAL target, resolves the
|
||||
// produced archive, and drives the self-restore-test (restore → boot → verify → teardown)
|
||||
// through the reconcile engine so it inherits the journal / per-guest serialization /
|
||||
// crash-safe recovery.
|
||||
//
|
||||
// Everything here is BENIGN (backup, restore-to-NEW, scratch teardown): it reuses the
|
||||
// slice-4 classifier/gate/journal via reconcile — no new destructive class, no new crypto.
|
||||
// Restore is to a NEW guest only (no overwrite this slice). PBS / offsite / zero-knowledge
|
||||
// is Phase B.
|
||||
//
|
||||
// Layout:
|
||||
// - runner.go — BackupRunner: vzdump + archive-volid/size resolve + the bulk-volume gap;
|
||||
// restore-candidate picker.
|
||||
// - store.go — in-memory latest-backup-per-target + latest-restore-test, implementing
|
||||
// the hub BackupReporter / RestoreTestReporter seams (point-in-time state
|
||||
// the collector reads; re-populated each cadence/selftest run).
|
||||
// - schedule.go — the restore-test cadence goroutine (default 24h; disabled when 0).
|
||||
//
|
||||
// hub does NOT import this package (the report types live in hub; this package imports hub
|
||||
// for them, mirroring the slice-5 storage seam). This package may import reconcile + hub +
|
||||
// proxmox (acyclic).
|
||||
package backup
|
||||
@@ -0,0 +1,198 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// BackupAPI is the read+backup proxmox surface the runner needs. *proxmox.Client satisfies it.
|
||||
type BackupAPI interface {
|
||||
Vzdump(ctx context.Context, opts proxmox.VzdumpOptions) (string, error)
|
||||
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
||||
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||||
StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error)
|
||||
}
|
||||
|
||||
// BackupRunner orchestrates a crash-consistent vzdump to a local target and reports the
|
||||
// result (incl. the bulk-volume gap). An agent-initiated vzdump is crash-consistent only
|
||||
// (no fsfreeze); the report says so.
|
||||
type BackupRunner struct {
|
||||
api BackupAPI
|
||||
target string // backup storage (content=backup)
|
||||
mode proxmox.BackupMode // default ModeSnapshot
|
||||
notes string // optional notes-template
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
|
||||
// for lvm-thin); the caller may pass ModeStop for storages without snapshot support.
|
||||
func NewBackupRunner(api BackupAPI, target string, mode proxmox.BackupMode, notes string, logger *slog.Logger) *BackupRunner {
|
||||
if mode == "" {
|
||||
mode = proxmox.ModeSnapshot
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, logger: logger, now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
// Backup runs one vzdump of vmid to the local target and returns the report record. A
|
||||
// failure is returned BOTH as an error and as a Backup{Success:false,...} so the caller can
|
||||
// record the failed attempt.
|
||||
func (r *BackupRunner) Backup(ctx context.Context, vmid int) (hub.Backup, error) {
|
||||
start := r.now()
|
||||
rec := hub.Backup{
|
||||
TargetID: r.target,
|
||||
VMID: vmid,
|
||||
Mode: string(r.mode),
|
||||
CrashConsistent: true, // always, this slice (no controller quiesce)
|
||||
StartedAt: start.Format(time.RFC3339),
|
||||
}
|
||||
if r.target == "" {
|
||||
rec.Error = "no backup target configured"
|
||||
return rec, fmt.Errorf("backup: %s", rec.Error)
|
||||
}
|
||||
|
||||
// Bulk-volume gap: which mountpoints the vzdump will EXCLUDE (best-effort; a config-read
|
||||
// failure just leaves the gap unknown, never fails the backup).
|
||||
if cfg, err := r.api.GuestConfig(ctx, vmid); err == nil {
|
||||
rec.UncoveredVolumes = uncoveredMountpoints(cfg.MountPoints())
|
||||
} else {
|
||||
r.logger.Warn("backup: could not read guest config for bulk-gap", "vmid", vmid, "err", err)
|
||||
rec.UncoveredVolumes = []string{}
|
||||
}
|
||||
|
||||
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
||||
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
||||
})
|
||||
if err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: 30 * time.Minute}); err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the produced archive (volid + size) — the task status carries no result volid.
|
||||
vol, size, err := r.latestArchive(ctx, vmid)
|
||||
if err != nil {
|
||||
rec.Error = fmt.Sprintf("backup succeeded but archive not resolved: %v", err)
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: resolve archive vmid %d: %w", vmid, err)
|
||||
}
|
||||
rec.Archive = vol
|
||||
rec.SizeBytes = size
|
||||
rec.Success = true
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
r.logger.Info("backup: completed", "vmid", vmid, "target", r.target, "archive", vol, "size_bytes", size,
|
||||
"uncovered_volumes", len(rec.UncoveredVolumes))
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// PickRestoreCandidate returns the newest backup archive on the target (any guest), or ""
|
||||
// when there is none — the restore-test then no-ops cleanly.
|
||||
func (r *BackupRunner) PickRestoreCandidate(ctx context.Context) (string, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var best string
|
||||
var bestCTime int64 = -1
|
||||
for _, e := range contents {
|
||||
if e.Content == "backup" && e.CTime > bestCTime {
|
||||
bestCTime, best = e.CTime, e.VolID
|
||||
}
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
// latestArchive finds the newest backup archive volid + size for vmid on the target.
|
||||
func (r *BackupRunner) latestArchive(ctx context.Context, vmid int) (string, int64, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var vol string
|
||||
var size, bestCTime int64 = 0, -1
|
||||
for _, e := range contents {
|
||||
if e.Content == "backup" && e.VMID == vmid && e.CTime > bestCTime {
|
||||
bestCTime, vol, size = e.CTime, e.VolID, e.Size
|
||||
}
|
||||
}
|
||||
if vol == "" {
|
||||
return "", 0, fmt.Errorf("no backup archive found for vmid %d on %s", vmid, r.target)
|
||||
}
|
||||
return vol, size, nil
|
||||
}
|
||||
|
||||
// uncoveredMountpoints returns the mountpoint paths the guest vzdump EXCLUDES. LXC mount
|
||||
// points are OPT-IN to vzdump: a mpN with `backup=1` is covered; ANY other state — the
|
||||
// `backup=` token absent OR `backup=0` — is excluded. We deliberately treat unset as
|
||||
// uncovered (the safe DR direction: never imply an unprotected volume is backed up).
|
||||
func uncoveredMountpoints(mps map[string]string) []string {
|
||||
var out []string
|
||||
for key, cfg := range mps {
|
||||
if mountpointCovered(cfg) {
|
||||
continue
|
||||
}
|
||||
out = append(out, mountpointLabel(key, cfg))
|
||||
}
|
||||
sort.Strings(out)
|
||||
if out == nil {
|
||||
return []string{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mountpointCovered reports whether a mpN config string is included in the vzdump — true
|
||||
// ONLY when it carries an explicit `backup=1`.
|
||||
func mountpointCovered(cfg string) bool {
|
||||
for _, tok := range strings.Split(cfg, ",") {
|
||||
if v, ok := strings.CutPrefix(tok, "backup="); ok {
|
||||
return v == "1"
|
||||
}
|
||||
}
|
||||
return false // no backup= token → opt-out by default → not covered
|
||||
}
|
||||
|
||||
// mountpointLabel prefers the mp=/path token, falling back to the mpN key.
|
||||
func mountpointLabel(key, cfg string) string {
|
||||
for _, tok := range strings.Split(cfg, ",") {
|
||||
if p, ok := strings.CutPrefix(tok, "mp="); ok && p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// ToHubRestoreTest maps a reconcile restore-test result to the hub wire record (the backup
|
||||
// package owns the reconcile→hub mapping so reconcile need not import hub for the result).
|
||||
func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.RestoreTest {
|
||||
rt := hub.RestoreTest{
|
||||
SourceArchive: res.Archive,
|
||||
SourceTier: res.SourceTier,
|
||||
ScratchVMID: res.ScratchVMID,
|
||||
Pass: res.Pass,
|
||||
Verified: res.Verified,
|
||||
TestedAt: testedAt.Format(time.RFC3339),
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
}
|
||||
if res.Err != nil {
|
||||
rt.Error = res.Err.Error()
|
||||
}
|
||||
return rt
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// RestoreTestRunner is the reconcile-engine seam the scheduler drives (*reconcile.Engine
|
||||
// satisfies it). Kept narrow so the scheduler is unit-testable with a fake.
|
||||
type RestoreTestRunner interface {
|
||||
RunRestoreTest(ctx context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult
|
||||
}
|
||||
|
||||
// CandidatePicker resolves the archive volid to restore-test (newest backup), or "" when
|
||||
// there is none yet (the tick then no-ops).
|
||||
type CandidatePicker func(ctx context.Context) (string, error)
|
||||
|
||||
// Scheduler runs the self-restore-test on an agent-internal cadence. It is the fourth daemon
|
||||
// goroutine; it does real restore→boot→destroy, so it only runs when the cadence is enabled
|
||||
// AND a valid scratch band is configured (validated by the caller before construction).
|
||||
type Scheduler struct {
|
||||
runner RestoreTestRunner
|
||||
pick CandidatePicker
|
||||
store *Store
|
||||
spec reconcile.RestoreTestSpec // archive is filled per-tick
|
||||
cadence time.Duration
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// SchedulerOptions configures a Scheduler.
|
||||
type SchedulerOptions struct {
|
||||
Runner RestoreTestRunner
|
||||
Pick CandidatePicker
|
||||
Store *Store
|
||||
Spec reconcile.RestoreTestSpec // RestoreStorage, ScratchMin/Max, SourceTier, BootTimeout
|
||||
Cadence time.Duration // 0 → disabled
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewScheduler builds a Scheduler.
|
||||
func NewScheduler(opts SchedulerOptions) *Scheduler {
|
||||
logger := opts.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Scheduler{
|
||||
runner: opts.Runner,
|
||||
pick: opts.Pick,
|
||||
store: opts.Store,
|
||||
spec: opts.Spec,
|
||||
cadence: opts.Cadence,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
|
||||
// Run fires a restore-test on the cadence until ctx is cancelled. A 0 cadence disables it
|
||||
// (the goroutine just waits for shutdown). It does NOT fire immediately on start (a restore
|
||||
// is heavy; the first runs one interval in) — on-demand runs use the selftest harness.
|
||||
// Returns nil on ctx cancellation.
|
||||
func (s *Scheduler) Run(ctx context.Context) error {
|
||||
if s.cadence <= 0 || s.runner == nil || s.pick == nil {
|
||||
s.logger.Info("backup: restore-test cadence disabled")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
s.logger.Info("backup: restore-test scheduler starting", "cadence", s.cadence)
|
||||
t := time.NewTicker(s.cadence)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.Info("backup: restore-test scheduler shutting down", "reason", ctx.Err())
|
||||
return nil
|
||||
case <-t.C:
|
||||
s.tick(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick runs one scheduled restore-test: pick a backup → run → record. No-ops cleanly when
|
||||
// no backup exists yet. Deterministic given s.now — tests call it directly.
|
||||
func (s *Scheduler) tick(ctx context.Context) {
|
||||
archive, err := s.pick(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: restore-test could not pick a candidate; skipping", "err", err)
|
||||
return
|
||||
}
|
||||
if archive == "" {
|
||||
s.logger.Info("backup: restore-test skipped; no backup available yet")
|
||||
return
|
||||
}
|
||||
spec := s.spec
|
||||
spec.Archive = archive
|
||||
res := s.runner.RunRestoreTest(ctx, spec)
|
||||
if res.Skipped {
|
||||
return // already logged by the engine (no free scratch VMID)
|
||||
}
|
||||
rt := ToHubRestoreTest(res, s.now())
|
||||
s.store.RecordRestoreTest(rt)
|
||||
if rt.Pass {
|
||||
s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
|
||||
} else {
|
||||
// A failing restore-test is the loudest DR signal there is.
|
||||
s.logger.Error("backup: scheduled restore-test FAILED", "archive", rt.SourceArchive, "err", rt.Error)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// Store holds the agent's LATEST backup result per target and the latest restore-test
|
||||
// result — the point-in-time state the host-report surfaces. It is updated by the backup
|
||||
// runner + the restore-test scheduler/selftest and read by the collector via the hub
|
||||
// BackupReporter / RestoreTestReporter seams. In-memory (lost on restart; the cadence
|
||||
// re-populates) and mutex-guarded for the concurrent collector vs scheduler access.
|
||||
type Store struct {
|
||||
mu sync.Mutex
|
||||
byTarget map[string]hub.Backup // latest backup per target id
|
||||
lastTest *hub.RestoreTest
|
||||
}
|
||||
|
||||
// NewStore builds an empty Store.
|
||||
func NewStore() *Store {
|
||||
return &Store{byTarget: map[string]hub.Backup{}}
|
||||
}
|
||||
|
||||
// RecordBackup stores the latest backup for its target.
|
||||
func (s *Store) RecordBackup(b hub.Backup) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.byTarget[b.TargetID] = b
|
||||
}
|
||||
|
||||
// RecordRestoreTest stores the latest restore-test result.
|
||||
func (s *Store) RecordRestoreTest(r hub.RestoreTest) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cp := r
|
||||
s.lastTest = &cp
|
||||
}
|
||||
|
||||
// Backups implements hub.BackupReporter — the latest backup per target (stable order by
|
||||
// target id is not guaranteed; the hub does not depend on order).
|
||||
func (s *Store) Backups(context.Context) []hub.Backup {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]hub.Backup, 0, len(s.byTarget))
|
||||
for _, b := range s.byTarget {
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RestoreTests implements hub.RestoreTestReporter — the latest restore-test result (0 or 1).
|
||||
func (s *Store) RestoreTests(context.Context) []hub.RestoreTest {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.lastTest == nil {
|
||||
return []hub.RestoreTest{}
|
||||
}
|
||||
return []hub.RestoreTest{*s.lastTest}
|
||||
}
|
||||
Reference in New Issue
Block a user