06e0bc9c25
The preventive counterpart to host_disk + storage_fill detectors: the periodic local whole-guest vzdump now prunes its own old archives (keep-last=3, clamped >=1) so a box can't refill its own root via its own backups. Local target only — PBS never pruned (resolved via ListStorage; fail-safe skip on unknown). Seeded in host-install. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs
230 lines
7.7 KiB
Go
230 lines
7.7 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
)
|
|
|
|
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
|
|
|
// fakeBackupAPI is a synthetic BackupAPI.
|
|
type fakeBackupAPI struct {
|
|
vzdumpUPID string
|
|
vzdumpErr error
|
|
waitErr error
|
|
cfg proxmox.GuestConfig
|
|
cfgErr error
|
|
content []proxmox.StorageContent
|
|
contentErr error
|
|
storages []proxmox.Storage // returned by ListStorage (the local-prune scope gate)
|
|
storageErr error
|
|
vzdumps []proxmox.VzdumpOptions
|
|
logLines []string // returned by TaskLogTail (e.g. "INFO: backup mode: stop")
|
|
waitGate chan struct{} // if non-nil, WaitTask blocks until closed (8B.2 watcher timing)
|
|
}
|
|
|
|
func (f *fakeBackupAPI) Vzdump(_ context.Context, o proxmox.VzdumpOptions) (string, error) {
|
|
f.vzdumps = append(f.vzdumps, o)
|
|
return f.vzdumpUPID, f.vzdumpErr
|
|
}
|
|
func (f *fakeBackupAPI) WaitTask(_ context.Context, _ string, _ proxmox.WaitOptions) (proxmox.TaskStatus, error) {
|
|
if f.waitGate != nil {
|
|
<-f.waitGate
|
|
}
|
|
return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, f.waitErr
|
|
}
|
|
func (f *fakeBackupAPI) GuestConfig(_ context.Context, _ int) (proxmox.GuestConfig, error) {
|
|
return f.cfg, f.cfgErr
|
|
}
|
|
func (f *fakeBackupAPI) StorageContent(_ context.Context, _ string) ([]proxmox.StorageContent, error) {
|
|
return f.content, f.contentErr
|
|
}
|
|
func (f *fakeBackupAPI) ListStorage(_ context.Context) ([]proxmox.Storage, error) {
|
|
return f.storages, f.storageErr
|
|
}
|
|
func (f *fakeBackupAPI) TaskLogTail(_ context.Context, _ string, _ int) ([]string, error) {
|
|
return f.logLines, nil
|
|
}
|
|
|
|
// guestCfgWithMounts builds a GuestConfig whose Extra carries the given mpN strings.
|
|
func guestCfgWithMounts(mps map[string]string) proxmox.GuestConfig {
|
|
extra := map[string]json.RawMessage{}
|
|
for k, v := range mps {
|
|
b, _ := json.Marshal(v)
|
|
extra[k] = b
|
|
}
|
|
return proxmox.GuestConfig{Extra: extra}
|
|
}
|
|
|
|
func TestBackup_SuccessResolvesArchiveAndBulkGap(t *testing.T) {
|
|
api := &fakeBackupAPI{
|
|
vzdumpUPID: "UPID:vzdump:1",
|
|
cfg: guestCfgWithMounts(map[string]string{
|
|
"mp0": "local-lvm:8,mp=/mnt/bulk,backup=0", // explicit opt-out → uncovered
|
|
"mp1": "local-lvm:4,mp=/mnt/db,backup=1", // covered
|
|
"mp2": "local-lvm:2,mp=/mnt/scratch", // UNSET → uncovered (opt-in default)
|
|
}),
|
|
content: []proxmox.StorageContent{
|
|
{VolID: "local:backup/old-9001.tar.zst", Content: "backup", VMID: 9001, Size: 100, CTime: 100},
|
|
{VolID: "local:backup/new-9001.tar.zst", Content: "backup", VMID: 9001, Size: 524288000, CTime: 200},
|
|
{VolID: "local:backup/other-9002.tar.zst", Content: "backup", VMID: 9002, Size: 7, CTime: 999},
|
|
},
|
|
}
|
|
r := NewBackupRunner(api, "local", "", "felhom test", "", quiet())
|
|
rec, err := r.Backup(context.Background(), 9001)
|
|
if err != nil {
|
|
t.Fatalf("Backup: %v", err)
|
|
}
|
|
if !rec.Success || !rec.CrashConsistent {
|
|
t.Errorf("record = %+v, want success + crash_consistent", rec)
|
|
}
|
|
if rec.Archive != "local:backup/new-9001.tar.zst" || rec.SizeBytes != 524288000 {
|
|
t.Errorf("resolved wrong archive/size: %+v", rec)
|
|
}
|
|
if rec.Mode != string(proxmox.ModeSnapshot) {
|
|
t.Errorf("mode = %q, want snapshot (default)", rec.Mode)
|
|
}
|
|
// Bulk gap: mp0 (backup=0) AND mp2 (unset) are uncovered; mp1 (backup=1) is NOT.
|
|
if got := rec.UncoveredVolumes; len(got) != 2 || !has(got, "/mnt/bulk") || !has(got, "/mnt/scratch") {
|
|
t.Errorf("uncovered = %v, want [/mnt/bulk /mnt/scratch] (unset is uncovered too)", got)
|
|
}
|
|
if has(rec.UncoveredVolumes, "/mnt/db") {
|
|
t.Error("backup=1 mountpoint must NOT be reported uncovered")
|
|
}
|
|
// Notes-template threaded through.
|
|
if len(api.vzdumps) != 1 || api.vzdumps[0].Notes != "felhom test" {
|
|
t.Errorf("vzdump opts = %+v", api.vzdumps)
|
|
}
|
|
}
|
|
|
|
func TestBackup_ReportsActualModeFromTaskLog(t *testing.T) {
|
|
// Requested snapshot, but PVE used stop (stopped guest) — the report must reflect ACTUAL.
|
|
api := &fakeBackupAPI{
|
|
vzdumpUPID: "UPID:vzdump:1",
|
|
content: []proxmox.StorageContent{{VolID: "v", Content: "backup", VMID: 9001, Size: 10, CTime: 1}},
|
|
logLines: []string{"INFO: CT Name: spike", "INFO: backup mode: stop", "INFO: Finished"},
|
|
}
|
|
r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", "", quiet())
|
|
rec, err := r.Backup(context.Background(), 9001)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rec.Mode != "stop" {
|
|
t.Errorf("mode = %q, want the ACTUAL %q from the task log (not the requested snapshot)", rec.Mode, "stop")
|
|
}
|
|
}
|
|
|
|
func TestBackup_VzdumpFailureReturnsFailedRecord(t *testing.T) {
|
|
api := &fakeBackupAPI{vzdumpErr: errors.New("vzdump boom")}
|
|
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
|
rec, err := r.Backup(context.Background(), 9001)
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
if rec.Success || rec.Error == "" {
|
|
t.Errorf("failed backup must produce a Success=false record with an Error: %+v", rec)
|
|
}
|
|
}
|
|
|
|
func TestPickRestoreCandidate_NewestOrEmpty(t *testing.T) {
|
|
api := &fakeBackupAPI{content: []proxmox.StorageContent{
|
|
{VolID: "a", Content: "backup", CTime: 10},
|
|
{VolID: "b", Content: "backup", CTime: 99},
|
|
{VolID: "iso", Content: "iso", CTime: 999}, // not a backup → ignored
|
|
}}
|
|
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
|
vol, err := r.PickRestoreCandidate(context.Background())
|
|
if err != nil || vol != "b" {
|
|
t.Fatalf("pick = %q,%v want newest 'b'", vol, err)
|
|
}
|
|
// no backups → "".
|
|
api.content = []proxmox.StorageContent{{VolID: "iso", Content: "iso"}}
|
|
if vol, _ := r.PickRestoreCandidate(context.Background()); vol != "" {
|
|
t.Errorf("no backup → empty, got %q", vol)
|
|
}
|
|
}
|
|
|
|
// --- scheduler ---
|
|
|
|
type fakeRTRunner struct {
|
|
res reconcile.RestoreTestResult
|
|
runs int
|
|
}
|
|
|
|
func (f *fakeRTRunner) RunRestoreTest(_ context.Context, _ reconcile.RestoreTestSpec) reconcile.RestoreTestResult {
|
|
f.runs++
|
|
return f.res
|
|
}
|
|
|
|
func TestScheduler_TickRunsAndRecords(t *testing.T) {
|
|
store := NewStore()
|
|
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Archive: "vol", Pass: true, Verified: "boot+running", Duration: time.Second}}
|
|
s := NewScheduler(SchedulerOptions{
|
|
Runner: rt,
|
|
Pick: func(context.Context) (string, error) { return "vol", nil },
|
|
Store: store,
|
|
Cadence: time.Hour,
|
|
Logger: quiet(),
|
|
})
|
|
s.tick(context.Background())
|
|
if rt.runs != 1 {
|
|
t.Fatalf("tick should run the restore-test once, got %d", rt.runs)
|
|
}
|
|
got := store.RestoreTests(context.Background())
|
|
if len(got) != 1 || !got[0].Pass || got[0].SourceArchive != "vol" {
|
|
t.Fatalf("store should have the recorded result: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestScheduler_TickNoBackupNoOp(t *testing.T) {
|
|
rt := &fakeRTRunner{}
|
|
s := NewScheduler(SchedulerOptions{
|
|
Runner: rt, Pick: func(context.Context) (string, error) { return "", nil },
|
|
Store: NewStore(), Cadence: time.Hour, Logger: quiet(),
|
|
})
|
|
s.tick(context.Background())
|
|
if rt.runs != 0 {
|
|
t.Errorf("no backup available → no restore-test run, got %d", rt.runs)
|
|
}
|
|
}
|
|
|
|
func TestScheduler_DisabledRunReturnsOnCancel(t *testing.T) {
|
|
s := NewScheduler(SchedulerOptions{Cadence: 0, Logger: quiet()})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- s.Run(ctx) }()
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
if err != nil {
|
|
t.Fatalf("disabled scheduler Run should return nil on cancel, got %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("disabled scheduler did not return on cancel")
|
|
}
|
|
}
|
|
|
|
func TestStore_Reporters(t *testing.T) {
|
|
s := NewStore()
|
|
if len(s.Backups(context.Background())) != 0 || len(s.RestoreTests(context.Background())) != 0 {
|
|
t.Fatal("empty store must report empty (non-nil) slices")
|
|
}
|
|
}
|
|
|
|
func has(ss []string, want string) bool {
|
|
for _, s := range ss {
|
|
if s == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|