R-366 slice 2 (decision 168): the host report carries the archives the restore-test skipped as another key's

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-07 10:18:50 +02:00
parent 7b0a8b234b
commit 2ac65c6210
7 changed files with 194 additions and 3 deletions
@@ -31,6 +31,7 @@ updates (and the old binary's capability probe would read `controllerswap-write`
 - Tests: wrapper `PVELane` (17; red first — the `pve-manager` plan was refused R12 on the old code), `pvegate` (5), `TestPVEGate_*` + `TestWritesEtcPVE`, `TestPVE_*`, `TestPVEHealthVerdict`, `TestPVEStepExecutor_*`. Red-proofs: `felhom.eu/documentation/audits/day-2026-10-07/B/`.
 ## Unreleased (2026-10-07)

+- R-366 slice 2 (`09` §3 decision 168): the restore-test pick records, per tier, the archives it skipped as written with another key (count, oldest, newest — no key material) in a `ForeignKeyLedger`; the host report carries it as `foreign_key_archives.tiers` (the stanza absent until a tier was evaluated since start, `tiers: []` when none — no null on the wire, the report contract forbids it). The hub turns a change into one operator line. Tests `TestR366_PickRecordsArchivesWrittenWithAnotherKey`, `TestR366_EvaluatedWithNoneIsAnEmptyList` (red-proved, `felhom.eu/documentation/audits/day-2026-10-07/E/`).
 - R-105 option A (`09` §3 decision 169): the `--selftest=escrow-create -directive <file>` flag and the escrow upload's `directive` field are removed — nothing read the directive; the DR path reads the recipe, tenantsync and the escrow blob. The hub ignores a `directive` from an older agent.

 ## v0.150.0 — the Docker step proves the engine reports a memory kill; after a restart the agent remembers the last backup per tier; three more SMART counters on the wire (R-528, R-894, R-330; `09` §3 decisions 157, 161) (2026-10-07)
@@ -790,6 +790,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 	pbsTargets := pbsTargetsFromPVE(cfg, px, logger)
 	pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargets, pbsStore, pbs.DefaultLiveSnapshotTimeout, logger)
 	collector := hub.NewCollector(px, newTunnelProber(cfg, px), observer, backupStore, backupStore, pbsReporter, cfg.Hub.HostID, version, logger)
+	// R-366 slice 2: the restore-test's ledger of archives written with another key → the host report.
+	foreignKeys := backup.NewForeignKeyLedger()
+	collector.SetForeignKeyArchiveReporter(foreignKeys)
 	collector.SetBackupTargetResolver(primaryBackupTargetOf(cfg)) // R-109: the recipe names the live target
 	// Privileged-capability self-check (v0.44.0): probe the sudoers grants the non-root agent
 	// depends on. The probe runs `sudo -n -l` LITERALLY (a policy LIST, never executing the
@@ -1015,7 +1018,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 	// with the local API so a backup and a restore-test can never run together.
 	rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
 	heavyOps := &backup.InFlight{}
-	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, logger)
+	scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, foreignKeys, logger)
 	// R-189: the host report's restore_tests[] must survive an agent restart. The in-memory store
 	// holds only this process's latest run, and under per-archive due-ness the agent will not
 	// re-test an archive it has already proven — so without this the hub can report a tier unproven
@@ -1707,7 +1710,7 @@ func primaryBackupTargetOf(cfg config.Config) func() hub.ConfiguredBackupTarget
 // disables the cadence (returns a scheduler that just waits) when the cadence is off or the
 // scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
 // machinery still works on-demand via --selftest=restore-test.
-func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, logger *slog.Logger) *backup.Scheduler {
+func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, foreign *backup.ForeignKeyLedger, logger *slog.Logger) *backup.Scheduler {
 	// R-86: this is the EVALUATION interval, not the trigger. What decides a test happens is the
 	// per-archive due-check in internal/backup/restoretest_due.go.
 	cadence := cfg.Backup.RestoreTestEvalInterval()
@@ -1726,6 +1729,9 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
 	min, max := cfg.Backup.ScratchBand()
 	target := cfg.Backup.BackupTarget()
 	runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
+	if foreign != nil {
+		runner.SetForeignKeyLedger(foreign) // R-366 slice 2: the pick records archives written with another key
+	}
 	// Every configured tier is a rotation candidate, not just the primary.
 	cfgTiers, _ := cfg.Backup.BackupTiers() // warnings already logged where the tiers are armed
 	tierIDs := make([]string, 0, len(cfgTiers))
@@ -2271,7 +2277,7 @@ func runSelftestRestoreTestDue(ctx context.Context, cfg config.Config, logger *s
 		return 1
 	}
 	rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
-	sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, logger)
+	sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, nil, logger)

 	fmt.Printf("eval_interval=%s settle=%s\n", cfg.Backup.RestoreTestEvalInterval(), cfg.Backup.RestoreTestSettle())
 	start := time.Now()
@@ -0,0 +1,60 @@
+package backup
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+)
+
+// ForeignKeyLedger (R-366 slice 2, `09` §3 decision 168) holds, per backup tier, the whole-guest archives the
+// restore-test pick skipped because another key wrote them (R-727 — an earlier install of this box). Since R-727 the
+// skip was one INFO log line per archive and nothing else, so after a reinstall the operator was never told that the
+// box's older whole-guest copies are unreadable to it. The host report carries this ledger; the hub raises ONE
+// operator event when it changes.
+//
+// It reports nil until a tier has been evaluated since the agent started, so a restart does not read as "the set
+// changed to empty" (the hub keeps its last state for an absent field).
+type ForeignKeyLedger struct {
+	mu       sync.Mutex
+	byTarget map[string]hub.ForeignKeyArchives
+}
+
+// NewForeignKeyLedger builds an empty ledger.
+func NewForeignKeyLedger() *ForeignKeyLedger {
+	return &ForeignKeyLedger{}
+}
+
+func (l *ForeignKeyLedger) set(target string, n int, oldest, newest int64) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.byTarget == nil {
+		l.byTarget = map[string]hub.ForeignKeyArchives{}
+	}
+	e := hub.ForeignKeyArchives{Target: target, Count: n}
+	if n > 0 {
+		e.Oldest = time.Unix(oldest, 0).UTC().Format(time.RFC3339)
+		e.Newest = time.Unix(newest, 0).UTC().Format(time.RFC3339)
+	}
+	l.byTarget[target] = e
+}
+
+// ForeignKeyArchives implements hub.ForeignKeyArchiveReporter: nil before any evaluation; otherwise the tiers that
+// hold such archives (`Tiers` empty, never nil, when none do), sorted by tier.
+func (l *ForeignKeyLedger) ForeignKeyArchives(context.Context) *hub.ForeignKeyArchivesStanza {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.byTarget == nil {
+		return nil
+	}
+	out := []hub.ForeignKeyArchives{}
+	for _, e := range l.byTarget {
+		if e.Count > 0 {
+			out = append(out, e)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Target < out[j].Target })
+	return &hub.ForeignKeyArchivesStanza{Tiers: out}
+}
@@ -0,0 +1,69 @@
+package backup
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// R-366 slice 2 (`09` §3 decision 168) — the restore-test's skip of an archive written with another key stops being
+// silent: the pick records, per tier, how many it skipped and their time range, and the host report carries it.
+//
+// COMPANION RED-PROOF (observed): remove the `r.foreign.set(...)` call from PickSettledRestoreCandidateOn → this fails
+// with "after one evaluation the ledger must report felhom-pbs: 2 archives …; got []". Restored.
+func TestR366_PickRecordsArchivesWrittenWithAnotherKey(t *testing.T) {
+	api := &fakeBackupAPI{
+		storages: []proxmox.Storage{{Storage: "felhom-pbs", Type: "pbs", EncryptionKey: thisBoxKey}},
+		content: []proxmox.StorageContent{
+			{VolID: "felhom-pbs:backup/ct/9201/2026-09-16T17:27:32Z", Content: "backup", VMID: 9201, Size: 4774114206, CTime: 1789579652, Encrypted: earlierBox2},
+			{VolID: "felhom-pbs:backup/ct/9201/2026-09-16T21:59:54Z", Content: "backup", VMID: 9201, Size: 20811501236, CTime: 1789595994, Encrypted: earlierBox1},
+			{VolID: "felhom-pbs:backup/ct/9201/2026-09-29T19:37:07Z", Content: "backup", VMID: 9201, Size: 3490689830, CTime: 1790710627, Encrypted: thisBoxKey},
+		},
+	}
+	r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", "keep-last=1", quiet())
+	l := NewForeignKeyLedger()
+	r.SetForeignKeyLedger(l)
+
+	if got := l.ForeignKeyArchives(context.Background()); got != nil {
+		t.Fatalf("before any evaluation the ledger must be nil (the hub keeps its state); got %v", got)
+	}
+	if _, _, err := r.PickSettledRestoreCandidateOn(context.Background(), "felhom-pbs", time.Time{}); err != nil {
+		t.Fatal(err)
+	}
+	st := l.ForeignKeyArchives(context.Background())
+	want := hub.ForeignKeyArchives{Target: "felhom-pbs", Count: 2, Oldest: "2026-09-16T17:27:32Z", Newest: "2026-09-16T21:59:54Z"}
+	var got []hub.ForeignKeyArchives
+	if st != nil {
+		got = st.Tiers
+	}
+	if len(got) != 1 || got[0] != want {
+		t.Fatalf("after one evaluation the ledger must report felhom-pbs: 2 archives 2026-09-16T17:27:32Z…21:59:54Z; got %v", got)
+	}
+}
+
+// Evaluated and none found → the stanza with `tiers: []`; not evaluated → no stanza at all. Never a null on the wire.
+func TestR366_EvaluatedWithNoneIsAnEmptyList(t *testing.T) {
+	api := &fakeBackupAPI{
+		storages: []proxmox.Storage{{Storage: "felhom-pbs", Type: "pbs", EncryptionKey: thisBoxKey}},
+		content:  []proxmox.StorageContent{{VolID: "felhom-pbs:backup/ct/9201/2026-09-29T19:37:07Z", Content: "backup", VMID: 9201, Size: 3490689830, CTime: 1790710627, Encrypted: thisBoxKey}},
+	}
+	r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", "keep-last=1", quiet())
+	l := NewForeignKeyLedger()
+	r.SetForeignKeyLedger(l)
+	b, _ := json.Marshal(hub.HostReport{ForeignKeyArchives: l.ForeignKeyArchives(context.Background())})
+	if strings.Contains(string(b), "foreign_key_archives") {
+		t.Fatalf("not evaluated must omit the stanza; got %s", b)
+	}
+	if _, _, err := r.PickSettledRestoreCandidateOn(context.Background(), "felhom-pbs", time.Time{}); err != nil {
+		t.Fatal(err)
+	}
+	b, _ = json.Marshal(hub.HostReport{ForeignKeyArchives: l.ForeignKeyArchives(context.Background())})
+	if !strings.Contains(string(b), `"foreign_key_archives":{"tiers":[]}`) {
+		t.Fatalf("evaluated with none must report tiers: []; got %s", b)
+	}
+}
@@ -66,8 +66,13 @@ type BackupRunner struct {
 	// due-check is served from the local-API handler goroutines.
 	rejectedMu sync.Mutex
 	rejected   map[string]struct{}
+	// foreign (R-366 slice 2) records, per tier, the archives the pick skipped as another key's. nil = not wired.
+	foreign *ForeignKeyLedger
 }

+// SetForeignKeyLedger wires the R-366 slice-2 ledger the host report reads.
+func (r *BackupRunner) SetForeignKeyLedger(l *ForeignKeyLedger) { r.foreign = l }
+
 // NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
 // for lvm-thin); the caller may pass ModeStop for storages without snapshot support. retention is the
 // per-run prune spec ("keep-last=N", or "" to never prune) — only the periodic local backup sets it.
@@ -370,6 +375,8 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target
 	var best string
 	var bestCTime int64 = -1
 	known := map[int]bool{} // vmid → the guest exists on this node (asked once per vmid per pick)
+	var foreignN int        // R-366 slice 2: archives skipped as another key's, and their time range
+	var foreignMin, foreignMax int64
 	for _, e := range contents {
 		if e.Content != "backup" {
 			continue
@@ -384,6 +391,13 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target
 		}
 		if ownKey != "" && !strings.EqualFold(e.Encrypted, ownKey) {
 			r.noteNotAGuestBackupOnce(e, fmt.Sprintf("written by another box (key %s, this box's key %s) — not this box's proof", shortFP(e.Encrypted), shortFP(ownKey)))
+			foreignN++
+			if foreignMin == 0 || e.CTime < foreignMin {
+				foreignMin = e.CTime
+			}
+			if e.CTime > foreignMax {
+				foreignMax = e.CTime
+			}
 			continue
 		}
 		// R-689 (v0.136.0): … OF A GUEST THAT STILL EXISTS here. Measured on demo-hp 2026-09-27 right after
@@ -420,6 +434,9 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target
 			bestCTime, best = e.CTime, e.VolID
 		}
 	}
+	if r.foreign != nil {
+		r.foreign.set(target, foreignN, foreignMin, foreignMax)
+	}
 	if best == "" {
 		return "", time.Time{}, nil
 	}
@@ -115,6 +115,7 @@ type Collector struct {
 	ctrlSup          ControllerSupervisorReporter                  // R-523: in-guest controller supervisor (nil → stanza omitted)
 	guestNet         GuestNetReporter                              // R-54: per-guest network watchdog (nil → stanza omitted)
 	diskTrim         GuestDiskTrimReporter                         // R-444: weekly guest disk trim (nil → stanza omitted)
+	foreignKey       ForeignKeyArchiveReporter                     // R-366 slice 2 (nil → omitted)
 	selfUpdate       SelfUpdateReporter                            // D1: agent self-update pending status (nil → false)
 	mgmtPlane        MgmtPlaneReporter                             // G1: management-plane health (nil → stanza omitted)
 	oob              OOBReporter                                   // H1: operator-access health (nil → stanza omitted)
@@ -228,6 +229,18 @@ func (c *Collector) SetGuestNetReporter(g GuestNetReporter) *Collector {
 	return c
 }

+// ForeignKeyArchiveReporter is the R-366 slice-2 seam: the restore-test's ledger of archives written with another
+// key. nil = not evaluated yet (the stanza is omitted and the hub keeps its state).
+type ForeignKeyArchiveReporter interface {
+	ForeignKeyArchives(ctx context.Context) *ForeignKeyArchivesStanza
+}
+
+// SetForeignKeyArchiveReporter wires the restore-test's foreign-key ledger (R-366 slice 2; nil-safe → omitted).
+func (c *Collector) SetForeignKeyArchiveReporter(r ForeignKeyArchiveReporter) *Collector {
+	c.foreignKey = r
+	return c
+}
+
 // SetGuestDiskTrimReporter wires the R-444 weekly trim job as a report source (nil-safe → stanza omitted).
 func (c *Collector) SetGuestDiskTrimReporter(r GuestDiskTrimReporter) *Collector {
 	c.diskTrim = r
@@ -394,6 +407,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
 	if c.diskTrim != nil {
 		report.GuestDiskTrim = c.diskTrim.GuestDiskTrimStatus(ctx)
 	}
+	// R-366 slice 2: archives the restore-test skipped as another key's (nil → not evaluated yet → omitted).
+	if c.foreignKey != nil {
+		report.ForeignKeyArchives = c.foreignKey.ForeignKeyArchives(ctx)
+	}
 	// D1: agent self-update pending status (nil reporter → pending=false, the steady state).
 	if c.selfUpdate != nil {
 		report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending()
@@ -129,6 +129,12 @@ type HostReport struct {
 	// wired; an empty `guests` list means the job runs and no guest has been trimmed yet. No secret.
 	GuestDiskTrim *GuestDiskTrimStatus `json:"guest_disk_trim,omitempty"`

+	// ForeignKeyArchives (R-366 slice 2, `09` §3 decision 168): per backup tier, the whole-guest archives the
+	// restore-test SKIPPED because they were written with another key (an earlier install of this box). This box
+	// cannot open them; the hub turns a CHANGE of this list into one operator event. Absent = not evaluated yet since
+	// the agent started (the hub keeps its last state); `tiers: []` = evaluated, none found.
+	ForeignKeyArchives *ForeignKeyArchivesStanza `json:"foreign_key_archives,omitempty"`
+
 	// LogTail is the agent's on-demand debug-ring tail (v0.83.0 observability) — the agent
 	// mirror of the controller's report log_tails channel. Present ONLY on the heartbeat
 	// right after the control envelope requested it (log_tail_requested); consume-once on
@@ -726,3 +732,18 @@ type WireRestoreDirective struct {
 	Archive string `json:"archive,omitempty"` // source archive/snapshot to restore from
 	VMID    int    `json:"vmid,omitempty"`
 }
+
+// ForeignKeyArchivesStanza wraps the per-tier list so "evaluated, none" (`tiers: []`) differs from "not evaluated"
+// (the stanza absent) without a null on the wire.
+type ForeignKeyArchivesStanza struct {
+	Tiers []ForeignKeyArchives `json:"tiers"`
+}
+
+// ForeignKeyArchives is one tier's count of archives written with another key (R-366 slice 2): the count and the
+// newest/oldest archive time (RFC3339, UTC). No key material — the fingerprints stay on the box.
+type ForeignKeyArchives struct {
+	Target string `json:"target"`
+	Count  int    `json:"count"`
+	Oldest string `json:"oldest"`
+	Newest string `json:"newest"`
+}