v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)

The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→ teardown) that closes "a backup you haven't restored isn't a backup". All benign (reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent. - proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID. - reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent). - internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence scheduler (4th daemon goroutine, default 24h) + in-memory report store. - hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical + bidirectional key-set tests; hub handler logs a FAILED restore-test prominently. - config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 13:49:39 +02:00
parent e548ab57fe
commit b527430ec7
23 changed files with 1727 additions and 46 deletions
@@ -31,12 +31,25 @@ type StorageObserver interface {
 	Observe(ctx context.Context) ([]StorageTarget, error)
 }

+// BackupReporter / RestoreTestReporter are the slice-6 seams the backup layer plugs into
+// (same consumer-side pattern as StorageObserver — hub does not import the backup package).
+// They return the agent's LATEST-known backup-per-target / restore-test result (point-in-time
+// state the backup layer accumulates), not a live scan. A nil reporter → empty slice.
+type BackupReporter interface {
+	Backups(ctx context.Context) []Backup
+}
+type RestoreTestReporter interface {
+	RestoreTests(ctx context.Context) []RestoreTest
+}
+
 // Collector builds a HostReport from read-only sources. All deps are behind narrow
 // interfaces for unit testing.
 type Collector struct {
 	px           proxmoxReader
 	cf           CloudflaredProber
 	storage      StorageObserver
+	backups      BackupReporter
+	restoreTests RestoreTestReporter
 	hostID       string
 	agentVersion string
 	logger       *slog.Logger
@@ -44,8 +57,8 @@ type Collector struct {
 }

 // NewCollector builds a collector. hostID echoes config.Hub.HostID; agentVersion is
-// the binary version. storage may be nil (storage_targets emitted empty).
-func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserver, hostID, agentVersion string, logger *slog.Logger) *Collector {
+// the binary version. storage/backups/restoreTests may be nil (their collections emit empty).
+func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserver, backups BackupReporter, restoreTests RestoreTestReporter, hostID, agentVersion string, logger *slog.Logger) *Collector {
 	if logger == nil {
 		logger = slog.Default()
 	}
@@ -53,6 +66,8 @@ func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserve
 		px:           px,
 		cf:           cf,
 		storage:      storage,
+		backups:      backups,
+		restoreTests: restoreTests,
 		hostID:       hostID,
 		agentVersion: agentVersion,
 		logger:       logger,
@@ -79,9 +94,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
 		// storage_targets populated this slice (slice 5) via the observer; the rest stay
 		// defined-but-empty (slice 6). Non-nil so they marshal as [].
 		StorageTargets: c.collectStorage(ctx),
-		Backups:        []Backup{},
-		RestoreTests:   []RestoreTest{},
-		PBSSnapshots:   []PBSSnapshot{},
+		Backups:        c.collectBackups(ctx),
+		RestoreTests:   c.collectRestoreTests(ctx),
+		PBSSnapshots:   []PBSSnapshot{}, // Phase B
+
 		AuditTail:      []AuditEntry{},
 		Cloudflared:    Cloudflared{Status: c.cloudflaredStatus(ctx)},
 	}
@@ -159,6 +175,29 @@ func (c *Collector) collectStorage(ctx context.Context) []StorageTarget {
 	return targets
 }

+// collectBackups / collectRestoreTests read the agent's latest backup + restore-test state
+// via the seams. Best-effort: a nil reporter or nil slice degrades to an empty (non-nil)
+// list so the collection always marshals as [].
+func (c *Collector) collectBackups(ctx context.Context) []Backup {
+	if c.backups == nil {
+		return []Backup{}
+	}
+	if b := c.backups.Backups(ctx); b != nil {
+		return b
+	}
+	return []Backup{}
+}
+
+func (c *Collector) collectRestoreTests(ctx context.Context) []RestoreTest {
+	if c.restoreTests == nil {
+		return []RestoreTest{}
+	}
+	if r := c.restoreTests.RestoreTests(ctx); r != nil {
+		return r
+	}
+	return []RestoreTest{}
+}
+
 func (c *Collector) cloudflaredStatus(ctx context.Context) string {
 	if c.cf == nil {
 		return "unknown"
@@ -33,7 +33,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) {
 	obs := fakeObserver{targets: []StorageTarget{
 		{Name: "local-lvm", Type: StorageTypeLVMThin, State: StorageStateAttached, Reachable: true},
 	}}
-	c := NewCollector(px, fakeProber{status: "active"}, obs, "h", "0.5.0", quietLogger())
+	c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, "h", "0.5.0", quietLogger())
 	r, err := c.Collect(context.Background())
 	if err != nil {
 		t.Fatalf("Collect: %v", err)
@@ -45,7 +45,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) {

 func TestCollect_StorageObserverErrorDegradesToEmpty(t *testing.T) {
 	px := &fakePx{node: "n", ns: newTestNodeStatus()}
-	c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, "h", "0.5.0", quietLogger())
+	c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, nil, nil, "h", "0.5.0", quietLogger())
 	r, err := c.Collect(context.Background())
 	if err != nil {
 		t.Fatalf("a storage observe error must not sink the heartbeat: %v", err)
@@ -64,7 +64,7 @@ func TestCollect_HostAndGuests(t *testing.T) {
 		},
 		cfg: map[int]proxmox.GuestConfig{100: {Cores: 2, Memory: 2048}},
 	}
-	c := NewCollector(px, fakeProber{status: "active"}, nil, "demo-host-01", "0.3.0", quietLogger())
+	c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "demo-host-01", "0.3.0", quietLogger())
 	r, err := c.Collect(context.Background())
 	if err != nil {
 		t.Fatalf("Collect: %v", err)
@@ -104,7 +104,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {
 		cfg:    map[int]proxmox.GuestConfig{100: {Cores: 2}},
 		cfgErr: map[int]error{200: errors.New("config read failed")},
 	}
-	c := NewCollector(px, fakeProber{status: "active"}, nil, "h", "0.3.1", quietLogger())
+	c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "h", "0.3.1", quietLogger())
 	r, err := c.Collect(context.Background())
 	if err != nil {
 		t.Fatalf("a per-guest failure must NOT fail the whole report: %v", err)
@@ -125,7 +125,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {

 func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {
 	px := &fakePx{node: "n", nsErr: errors.New("proxmox down")}
-	c := NewCollector(px, fakeProber{status: "active"}, nil, "h", "0.3.0", quietLogger())
+	c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, "h", "0.3.0", quietLogger())
 	if _, err := c.Collect(context.Background()); err == nil {
 		t.Fatal("NodeStatus failure must be a hard error (no useful report)")
 	}
@@ -133,7 +133,7 @@ func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {

 func TestCollect_CloudflaredProbeErrorIsUnknown(t *testing.T) {
 	px := &fakePx{node: "n", ns: newTestNodeStatus()}
-	c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, "h", "0.3.0", quietLogger())
+	c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, nil, nil, "h", "0.3.0", quietLogger())
 	r, err := c.Collect(context.Background())
 	if err != nil {
 		t.Fatalf("cloudflared failure must not be fatal: %v", err)
@@ -46,7 +46,19 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
 				Smart: SmartSummary{Health: SmartUnknown},
 			},
 		},
-		Backups: []Backup{}, RestoreTests: []RestoreTest{},
+		Backups: []Backup{
+			{
+				TargetID: "local", VMID: 9001, Archive: "local:backup/x.tar.zst", Mode: "snapshot",
+				CrashConsistent: true, SizeBytes: 1, Success: true, StartedAt: "2026-06-09T11:00:00Z",
+				DurationSeconds: 1, UncoveredVolumes: []string{"/mnt/bulk"},
+			},
+		},
+		RestoreTests: []RestoreTest{
+			{
+				SourceArchive: "local:backup/x.tar.zst", SourceTier: "local", ScratchVMID: 990000,
+				Pass: true, Verified: "boot+running", TestedAt: "2026-06-09T11:05:00Z", DurationSeconds: 1,
+			},
+		},
 		PBSSnapshots: []PBSSnapshot{}, AuditTail: []AuditEntry{},
 		Cloudflared:  Cloudflared{Status: "active"},
 	}
@@ -66,6 +78,10 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
 	assertSameKeys(t, "storage_targets[0]", gst, sst)
 	assertSameKeys(t, "storage_targets[0].smart", field(gst, "smart"), field(sst, "smart"))
 	assertSameKeys(t, "storage_targets[0].thin_pool", field(gst, "thin_pool"), field(sst, "thin_pool"))
+
+	// slice-6 additions — backups[0] / restore_tests[0] key sets (the bidirectional guard).
+	assertSameKeys(t, "backups[0]", firstElem(golden["backups"]), firstElem(got["backups"]))
+	assertSameKeys(t, "restore_tests[0]", firstElem(golden["restore_tests"]), firstElem(got["restore_tests"]))
 }

 // field extracts a nested object value from a decoded JSON map (nil if absent/not a map).
@@ -165,10 +165,41 @@ const (
 	StorageStateDecommissioned = "decommissioned"
 )

-type Backup struct{} // slice 6: per-target backup status fields TBD
-type RestoreTest struct{}   // slice 6: self-restore-test result fields TBD
-type PBSSnapshot struct{}   // slice 6: PBS snapshot inventory fields TBD
-type AuditEntry struct{}    // audit-log tail entry fields TBD
+// Backup is the latest guest-vzdump result per target (doc 03 §8, slice 6 Phase A). An
+// agent-initiated vzdump is CRASH-CONSISTENT only (no fsfreeze; app-consistency needs the
+// controller quiesce, slice 8) — marked so here. UncoveredVolumes lists the guest's
+// backup=0 (or backup-unset) mountpoints excluded from the vzdump — the bulk-volume DR gap
+// (the bulk-backup mechanism is slice 10). Cross-repo contract: keep byte-identical with
+// felhom.eu/hub and the bidirectional golden key-set test.
+type Backup struct {
+	TargetID         string   `json:"target_id"`         // backup storage name (e.g. "local")
+	VMID             int      `json:"vmid"`              // source guest
+	Archive          string   `json:"archive"`           // produced vzdump volid
+	Mode             string   `json:"mode"`              // snapshot | stop
+	CrashConsistent  bool     `json:"crash_consistent"`  // always true this slice
+	SizeBytes        int64    `json:"size_bytes"`
+	Success          bool     `json:"success"`
+	Error            string   `json:"error,omitempty"`
+	StartedAt        string   `json:"started_at"`        // RFC3339
+	DurationSeconds  float64  `json:"duration_seconds"`
+	UncoveredVolumes []string `json:"uncovered_volumes"` // backup=0/unset mountpoints (bulk gap)
+}
+
+// RestoreTest is the latest self-restore-test result (doc 03 §8). This slice verifies
+// boot+running only (deep app-health is slice 8). SourceTier is "local" here; PBS is Phase B.
+type RestoreTest struct {
+	SourceArchive   string  `json:"source_archive"`
+	SourceTier      string  `json:"source_tier"` // "local" (pbs = Phase B)
+	ScratchVMID     int     `json:"scratch_vmid"`
+	Pass            bool    `json:"pass"`
+	Verified        string  `json:"verified"` // "boot+running" this slice
+	Error           string  `json:"error,omitempty"`
+	TestedAt        string  `json:"tested_at"` // RFC3339
+	DurationSeconds float64 `json:"duration_seconds"`
+}
+
+type PBSSnapshot struct{} // slice 6 Phase B: PBS snapshot inventory fields TBD
+type AuditEntry struct{}  // audit-log tail entry fields TBD

 // ControlEnvelope is the hub's 200 response to a host-report. This slice the agent
 // adopts ONLY PollIntervalSeconds; the rest are reserved/forward-compat fields it
@@ -86,8 +86,31 @@
      }
    }
  ],
-  "backups": [],
-  "restore_tests": [],
+  "backups": [
+    {
+      "target_id": "local",
+      "vmid": 9001,
+      "archive": "local:backup/vzdump-lxc-9001-2026_06_09-11_00_00.tar.zst",
+      "mode": "snapshot",
+      "crash_consistent": true,
+      "size_bytes": 524288000,
+      "success": true,
+      "started_at": "2026-06-09T11:00:00Z",
+      "duration_seconds": 42.5,
+      "uncovered_volumes": ["/mnt/bulk"]
+    }
+  ],
+  "restore_tests": [
+    {
+      "source_archive": "local:backup/vzdump-lxc-9001-2026_06_09-11_00_00.tar.zst",
+      "source_tier": "local",
+      "scratch_vmid": 990000,
+      "pass": true,
+      "verified": "boot+running",
+      "tested_at": "2026-06-09T11:05:00Z",
+      "duration_seconds": 38.2
+    }
+  ],
  "pbs_snapshots": [],
  "cloudflared": { "status": "active" },
  "audit_tail": []