v0.6.0-rc1: slice 6 Phase A — backup + the self-restore-test (local target)

The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→
teardown) that closes "a backup you haven't restored isn't a backup". All benign
(reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local
target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent.

- proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID.
- reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down
  pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a
  leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent).
- internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence
  scheduler (4th daemon goroutine, default 24h) + in-memory report store.
- hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical +
  bidirectional key-set tests; hub handler logs a FAILED restore-test prominently.
- config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

internal/reconcile/recover.go

@@ -3,6 +3,8 @@ package reconcile
 import (
 	"context"
 	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
 )
 
 // Recover consumes the journal's in-flight set at startup: resume-or-rollback for any
@@ -32,6 +34,16 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
 	}
 	for _, entry := range e.journal.InFlight() {
 		res.Examined++
+
+		// Scratch entries (slice-6 restore-test) are resolved by TEARDOWN, not by
+		// re-checking a sub-task UPID — a leaked scratch guest is the failure mode that
+		// matters. Handle them BEFORE the generic UPID path (else the restore sub-task's OK
+		// status would mark the entry succeeded while the guest still exists → leak).
+		if entry.Scratch {
+			e.recoverScratch(ctx, entry, &res)
+			continue
+		}
+
 		if entry.UPID == "" {
 			// POST never confirmed → abandon (fail-safe).
 			e.append(terminal(entry, OpFailed))
@@ -72,18 +84,80 @@ func (e *Engine) Recover(ctx context.Context) RecoverResult {
 	return res
 }
 
+// recoverScratch resolves a leaked restore-test scratch guest (slice 6, doc 03 §8/§10).
+// The invariant: a Scratch entry in-flight at startup means "scratch guest VMID may exist
+// and must be destroyed." It is idempotent — if the guest is already gone (crash after the
+// destroy task but before the terminal record), it records terminal-clean. The teardown
+// routes through the gate as a benign ClassGuestDestroy (agent-tagged scratch provenance) —
+// the same audit-bearing path the normal teardown uses.
+func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *RecoverResult) {
+	lxc, err := e.api.ListLXC(ctx)
+	if err != nil {
+		// Can't tell whether the guest exists → leave in-flight; a later Recover retries.
+		res.Unresolved++
+		e.logger.Warn("recover: cannot list guests to resolve leaked scratch; left in-flight",
+			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
+		return
+	}
+	exists := false
+	for _, g := range lxc {
+		if g.VMID == entry.VMID {
+			exists = true
+			break
+		}
+	}
+	if !exists {
+		// Already gone (idempotent) → the scratch left no leak.
+		e.append(terminal(entry, OpSucceeded))
+		res.ScratchClean++
+		e.logger.Info("recover: leaked-scratch entry resolved; guest already gone",
+			"op_id", entry.OpID, "vmid", entry.VMID)
+		return
+	}
+
+	dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, entry.VMID), nil)
+	if !dec.Allowed {
+		// Should be benign; if not, fail-safe (leave in-flight, do NOT force a destroy).
+		res.Unresolved++
+		e.logger.Error("recover: scratch teardown refused by gate (unexpected); left in-flight",
+			"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
+		return
+	}
+	upid, err := e.api.DestroyLXC(ctx, entry.VMID)
+	if err != nil {
+		res.Unresolved++
+		e.logger.Warn("recover: destroying leaked scratch failed; left in-flight (will retry)",
+			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
+		return
+	}
+	if upid != "" {
+		if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
+			res.Unresolved++
+			e.logger.Warn("recover: leaked-scratch destroy task failed; left in-flight (will retry)",
+				"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
+			return
+		}
+	}
+	e.append(terminal(entry, OpSucceeded))
+	res.ScratchDestroyed++
+	e.logger.Warn("recover: destroyed leaked restore-test scratch guest",
+		"op_id", entry.OpID, "vmid", entry.VMID)
+}
+
 // RecoverResult summarizes a startup recovery pass.
 type RecoverResult struct {
-	Examined     int
-	Resumed      int // task found completed OK and recorded succeeded
-	Failed       int // task found ended non-OK and recorded failed
-	RolledBack   int // no task id → abandoned (fail-safe)
-	StillRunning int // task still executing → left in-flight
-	Unresolved   int // task status unreadable → left in-flight
+	Examined         int
+	Resumed          int // task found completed OK and recorded succeeded
+	Failed           int // task found ended non-OK and recorded failed
+	RolledBack       int // no task id → abandoned (fail-safe)
+	StillRunning     int // task still executing → left in-flight
+	Unresolved       int // task status unreadable → left in-flight
+	ScratchClean     int // scratch entry resolved: guest already gone (no leak)
+	ScratchDestroyed int // scratch entry resolved: leaked guest destroyed
 }
 
 // terminal builds a terminal journal record preserving the op's identity, with the
-// idempotency key carried through so a SUCCEEDED one-shot op marks its key applied.
+// idempotency key + scratch flag carried through.
 func terminal(e JournalEntry, state OpState) JournalEntry {
 	return JournalEntry{
 		OpID:     e.OpID,
@@ -92,6 +166,7 @@ func terminal(e JournalEntry, state OpState) JournalEntry {
 		UPID:     e.UPID,
 		State:    state,
 		IdempKey: e.IdempKey,
+		Scratch:  e.Scratch,
 		At:       time.Now().UTC(),
 	}
 }