diff --git a/CHANGELOG.md b/CHANGELOG.md index 75fd337..d8226e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ updates (and the old binary's capability probe would read `controllerswap-write` - Tests: wrapper `PVELane` (17; red first — the `pve-manager` plan was refused R12 on the old code), `pvegate` (5), `TestPVEGate_*` + `TestWritesEtcPVE`, `TestPVE_*`, `TestPVEHealthVerdict`, `TestPVEStepExecutor_*`. Red-proofs: `felhom.eu/documentation/audits/day-2026-10-07/B/`. ## Unreleased (2026-10-07) +- R-366 slice 2 (`09` §3 decision 168): the restore-test pick records, per tier, the archives it skipped as written with another key (count, oldest, newest — no key material) in a `ForeignKeyLedger`; the host report carries it as `foreign_key_archives.tiers` (the stanza absent until a tier was evaluated since start, `tiers: []` when none — no null on the wire, the report contract forbids it). The hub turns a change into one operator line. Tests `TestR366_PickRecordsArchivesWrittenWithAnotherKey`, `TestR366_EvaluatedWithNoneIsAnEmptyList` (red-proved, `felhom.eu/documentation/audits/day-2026-10-07/E/`). - R-105 option A (`09` §3 decision 169): the `--selftest=escrow-create -directive ` flag and the escrow upload's `directive` field are removed — nothing read the directive; the DR path reads the recipe, tenantsync and the escrow blob. The hub ignores a `directive` from an older agent. ## v0.150.0 — the Docker step proves the engine reports a memory kill; after a restart the agent remembers the last backup per tier; three more SMART counters on the wire (R-528, R-894, R-330; `09` §3 decisions 157, 161) (2026-10-07) diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 3eb5a85..ef60639 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -790,6 +790,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int pbsTargets := pbsTargetsFromPVE(cfg, px, logger) pbsReporter := pbs.NewLiveSnapshotReporter(pbsTargets, pbsStore, pbs.DefaultLiveSnapshotTimeout, logger) collector := hub.NewCollector(px, newTunnelProber(cfg, px), observer, backupStore, backupStore, pbsReporter, cfg.Hub.HostID, version, logger) + // R-366 slice 2: the restore-test's ledger of archives written with another key → the host report. + foreignKeys := backup.NewForeignKeyLedger() + collector.SetForeignKeyArchiveReporter(foreignKeys) collector.SetBackupTargetResolver(primaryBackupTargetOf(cfg)) // R-109: the recipe names the live target // Privileged-capability self-check (v0.44.0): probe the sudoers grants the non-root agent // depends on. The probe runs `sudo -n -l` LITERALLY (a policy LIST, never executing the @@ -1015,7 +1018,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int // with the local API so a backup and a restore-test can never run together. rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json")) heavyOps := &backup.InFlight{} - scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, logger) + scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, foreignKeys, logger) // R-189: the host report's restore_tests[] must survive an agent restart. The in-memory store // holds only this process's latest run, and under per-archive due-ness the agent will not // re-test an archive it has already proven — so without this the hub can report a tier unproven @@ -1707,7 +1710,7 @@ func primaryBackupTargetOf(cfg config.Config) func() hub.ConfiguredBackupTarget // disables the cadence (returns a scheduler that just waits) when the cadence is off or the // scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the // machinery still works on-demand via --selftest=restore-test. -func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, logger *slog.Logger) *backup.Scheduler { +func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, foreign *backup.ForeignKeyLedger, logger *slog.Logger) *backup.Scheduler { // R-86: this is the EVALUATION interval, not the trigger. What decides a test happens is the // per-archive due-check in internal/backup/restoretest_due.go. cadence := cfg.Backup.RestoreTestEvalInterval() @@ -1726,6 +1729,9 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re min, max := cfg.Backup.ScratchBand() target := cfg.Backup.BackupTarget() runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger) + if foreign != nil { + runner.SetForeignKeyLedger(foreign) // R-366 slice 2: the pick records archives written with another key + } // Every configured tier is a rotation candidate, not just the primary. cfgTiers, _ := cfg.Backup.BackupTiers() // warnings already logged where the tiers are armed tierIDs := make([]string, 0, len(cfgTiers)) @@ -2271,7 +2277,7 @@ func runSelftestRestoreTestDue(ctx context.Context, cfg config.Config, logger *s return 1 } rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json")) - sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, logger) + sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, nil, logger) fmt.Printf("eval_interval=%s settle=%s\n", cfg.Backup.RestoreTestEvalInterval(), cfg.Backup.RestoreTestSettle()) start := time.Now() diff --git a/internal/backup/foreign_key_ledger.go b/internal/backup/foreign_key_ledger.go new file mode 100644 index 0000000..7791abd --- /dev/null +++ b/internal/backup/foreign_key_ledger.go @@ -0,0 +1,60 @@ +package backup + +import ( + "context" + "sort" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// ForeignKeyLedger (R-366 slice 2, `09` §3 decision 168) holds, per backup tier, the whole-guest archives the +// restore-test pick skipped because another key wrote them (R-727 — an earlier install of this box). Since R-727 the +// skip was one INFO log line per archive and nothing else, so after a reinstall the operator was never told that the +// box's older whole-guest copies are unreadable to it. The host report carries this ledger; the hub raises ONE +// operator event when it changes. +// +// It reports nil until a tier has been evaluated since the agent started, so a restart does not read as "the set +// changed to empty" (the hub keeps its last state for an absent field). +type ForeignKeyLedger struct { + mu sync.Mutex + byTarget map[string]hub.ForeignKeyArchives +} + +// NewForeignKeyLedger builds an empty ledger. +func NewForeignKeyLedger() *ForeignKeyLedger { + return &ForeignKeyLedger{} +} + +func (l *ForeignKeyLedger) set(target string, n int, oldest, newest int64) { + l.mu.Lock() + defer l.mu.Unlock() + if l.byTarget == nil { + l.byTarget = map[string]hub.ForeignKeyArchives{} + } + e := hub.ForeignKeyArchives{Target: target, Count: n} + if n > 0 { + e.Oldest = time.Unix(oldest, 0).UTC().Format(time.RFC3339) + e.Newest = time.Unix(newest, 0).UTC().Format(time.RFC3339) + } + l.byTarget[target] = e +} + +// ForeignKeyArchives implements hub.ForeignKeyArchiveReporter: nil before any evaluation; otherwise the tiers that +// hold such archives (`Tiers` empty, never nil, when none do), sorted by tier. +func (l *ForeignKeyLedger) ForeignKeyArchives(context.Context) *hub.ForeignKeyArchivesStanza { + l.mu.Lock() + defer l.mu.Unlock() + if l.byTarget == nil { + return nil + } + out := []hub.ForeignKeyArchives{} + for _, e := range l.byTarget { + if e.Count > 0 { + out = append(out, e) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Target < out[j].Target }) + return &hub.ForeignKeyArchivesStanza{Tiers: out} +} diff --git a/internal/backup/r366_foreign_key_ledger_test.go b/internal/backup/r366_foreign_key_ledger_test.go new file mode 100644 index 0000000..83304b7 --- /dev/null +++ b/internal/backup/r366_foreign_key_ledger_test.go @@ -0,0 +1,69 @@ +package backup + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// R-366 slice 2 (`09` §3 decision 168) — the restore-test's skip of an archive written with another key stops being +// silent: the pick records, per tier, how many it skipped and their time range, and the host report carries it. +// +// COMPANION RED-PROOF (observed): remove the `r.foreign.set(...)` call from PickSettledRestoreCandidateOn → this fails +// with "after one evaluation the ledger must report felhom-pbs: 2 archives …; got []". Restored. +func TestR366_PickRecordsArchivesWrittenWithAnotherKey(t *testing.T) { + api := &fakeBackupAPI{ + storages: []proxmox.Storage{{Storage: "felhom-pbs", Type: "pbs", EncryptionKey: thisBoxKey}}, + content: []proxmox.StorageContent{ + {VolID: "felhom-pbs:backup/ct/9201/2026-09-16T17:27:32Z", Content: "backup", VMID: 9201, Size: 4774114206, CTime: 1789579652, Encrypted: earlierBox2}, + {VolID: "felhom-pbs:backup/ct/9201/2026-09-16T21:59:54Z", Content: "backup", VMID: 9201, Size: 20811501236, CTime: 1789595994, Encrypted: earlierBox1}, + {VolID: "felhom-pbs:backup/ct/9201/2026-09-29T19:37:07Z", Content: "backup", VMID: 9201, Size: 3490689830, CTime: 1790710627, Encrypted: thisBoxKey}, + }, + } + r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", "keep-last=1", quiet()) + l := NewForeignKeyLedger() + r.SetForeignKeyLedger(l) + + if got := l.ForeignKeyArchives(context.Background()); got != nil { + t.Fatalf("before any evaluation the ledger must be nil (the hub keeps its state); got %v", got) + } + if _, _, err := r.PickSettledRestoreCandidateOn(context.Background(), "felhom-pbs", time.Time{}); err != nil { + t.Fatal(err) + } + st := l.ForeignKeyArchives(context.Background()) + want := hub.ForeignKeyArchives{Target: "felhom-pbs", Count: 2, Oldest: "2026-09-16T17:27:32Z", Newest: "2026-09-16T21:59:54Z"} + var got []hub.ForeignKeyArchives + if st != nil { + got = st.Tiers + } + if len(got) != 1 || got[0] != want { + t.Fatalf("after one evaluation the ledger must report felhom-pbs: 2 archives 2026-09-16T17:27:32Z…21:59:54Z; got %v", got) + } +} + +// Evaluated and none found → the stanza with `tiers: []`; not evaluated → no stanza at all. Never a null on the wire. +func TestR366_EvaluatedWithNoneIsAnEmptyList(t *testing.T) { + api := &fakeBackupAPI{ + storages: []proxmox.Storage{{Storage: "felhom-pbs", Type: "pbs", EncryptionKey: thisBoxKey}}, + content: []proxmox.StorageContent{{VolID: "felhom-pbs:backup/ct/9201/2026-09-29T19:37:07Z", Content: "backup", VMID: 9201, Size: 3490689830, CTime: 1790710627, Encrypted: thisBoxKey}}, + } + r := NewBackupRunner(api, "local", proxmox.ModeSnapshot, "", "keep-last=1", quiet()) + l := NewForeignKeyLedger() + r.SetForeignKeyLedger(l) + b, _ := json.Marshal(hub.HostReport{ForeignKeyArchives: l.ForeignKeyArchives(context.Background())}) + if strings.Contains(string(b), "foreign_key_archives") { + t.Fatalf("not evaluated must omit the stanza; got %s", b) + } + if _, _, err := r.PickSettledRestoreCandidateOn(context.Background(), "felhom-pbs", time.Time{}); err != nil { + t.Fatal(err) + } + b, _ = json.Marshal(hub.HostReport{ForeignKeyArchives: l.ForeignKeyArchives(context.Background())}) + if !strings.Contains(string(b), `"foreign_key_archives":{"tiers":[]}`) { + t.Fatalf("evaluated with none must report tiers: []; got %s", b) + } +} diff --git a/internal/backup/runner.go b/internal/backup/runner.go index c9767b0..66d5cf9 100644 --- a/internal/backup/runner.go +++ b/internal/backup/runner.go @@ -66,8 +66,13 @@ type BackupRunner struct { // due-check is served from the local-API handler goroutines. rejectedMu sync.Mutex rejected map[string]struct{} + // foreign (R-366 slice 2) records, per tier, the archives the pick skipped as another key's. nil = not wired. + foreign *ForeignKeyLedger } +// SetForeignKeyLedger wires the R-366 slice-2 ledger the host report reads. +func (r *BackupRunner) SetForeignKeyLedger(l *ForeignKeyLedger) { r.foreign = l } + // NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and // for lvm-thin); the caller may pass ModeStop for storages without snapshot support. retention is the // per-run prune spec ("keep-last=N", or "" to never prune) — only the periodic local backup sets it. @@ -370,6 +375,8 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target var best string var bestCTime int64 = -1 known := map[int]bool{} // vmid → the guest exists on this node (asked once per vmid per pick) + var foreignN int // R-366 slice 2: archives skipped as another key's, and their time range + var foreignMin, foreignMax int64 for _, e := range contents { if e.Content != "backup" { continue @@ -384,6 +391,13 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target } if ownKey != "" && !strings.EqualFold(e.Encrypted, ownKey) { r.noteNotAGuestBackupOnce(e, fmt.Sprintf("written by another box (key %s, this box's key %s) — not this box's proof", shortFP(e.Encrypted), shortFP(ownKey))) + foreignN++ + if foreignMin == 0 || e.CTime < foreignMin { + foreignMin = e.CTime + } + if e.CTime > foreignMax { + foreignMax = e.CTime + } continue } // R-689 (v0.136.0): … OF A GUEST THAT STILL EXISTS here. Measured on demo-hp 2026-09-27 right after @@ -420,6 +434,9 @@ func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target bestCTime, best = e.CTime, e.VolID } } + if r.foreign != nil { + r.foreign.set(target, foreignN, foreignMin, foreignMax) + } if best == "" { return "", time.Time{}, nil } diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 3098ec0..f1c5c12 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -115,6 +115,7 @@ type Collector struct { ctrlSup ControllerSupervisorReporter // R-523: in-guest controller supervisor (nil → stanza omitted) guestNet GuestNetReporter // R-54: per-guest network watchdog (nil → stanza omitted) diskTrim GuestDiskTrimReporter // R-444: weekly guest disk trim (nil → stanza omitted) + foreignKey ForeignKeyArchiveReporter // R-366 slice 2 (nil → omitted) selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false) mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted) oob OOBReporter // H1: operator-access health (nil → stanza omitted) @@ -228,6 +229,18 @@ func (c *Collector) SetGuestNetReporter(g GuestNetReporter) *Collector { return c } +// ForeignKeyArchiveReporter is the R-366 slice-2 seam: the restore-test's ledger of archives written with another +// key. nil = not evaluated yet (the stanza is omitted and the hub keeps its state). +type ForeignKeyArchiveReporter interface { + ForeignKeyArchives(ctx context.Context) *ForeignKeyArchivesStanza +} + +// SetForeignKeyArchiveReporter wires the restore-test's foreign-key ledger (R-366 slice 2; nil-safe → omitted). +func (c *Collector) SetForeignKeyArchiveReporter(r ForeignKeyArchiveReporter) *Collector { + c.foreignKey = r + return c +} + // SetGuestDiskTrimReporter wires the R-444 weekly trim job as a report source (nil-safe → stanza omitted). func (c *Collector) SetGuestDiskTrimReporter(r GuestDiskTrimReporter) *Collector { c.diskTrim = r @@ -394,6 +407,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { if c.diskTrim != nil { report.GuestDiskTrim = c.diskTrim.GuestDiskTrimStatus(ctx) } + // R-366 slice 2: archives the restore-test skipped as another key's (nil → not evaluated yet → omitted). + if c.foreignKey != nil { + report.ForeignKeyArchives = c.foreignKey.ForeignKeyArchives(ctx) + } // D1: agent self-update pending status (nil reporter → pending=false, the steady state). if c.selfUpdate != nil { report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending() diff --git a/internal/hub/report.go b/internal/hub/report.go index 48bc6b1..2dbc971 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -129,6 +129,12 @@ type HostReport struct { // wired; an empty `guests` list means the job runs and no guest has been trimmed yet. No secret. GuestDiskTrim *GuestDiskTrimStatus `json:"guest_disk_trim,omitempty"` + // ForeignKeyArchives (R-366 slice 2, `09` §3 decision 168): per backup tier, the whole-guest archives the + // restore-test SKIPPED because they were written with another key (an earlier install of this box). This box + // cannot open them; the hub turns a CHANGE of this list into one operator event. Absent = not evaluated yet since + // the agent started (the hub keeps its last state); `tiers: []` = evaluated, none found. + ForeignKeyArchives *ForeignKeyArchivesStanza `json:"foreign_key_archives,omitempty"` + // LogTail is the agent's on-demand debug-ring tail (v0.83.0 observability) — the agent // mirror of the controller's report log_tails channel. Present ONLY on the heartbeat // right after the control envelope requested it (log_tail_requested); consume-once on @@ -726,3 +732,18 @@ type WireRestoreDirective struct { Archive string `json:"archive,omitempty"` // source archive/snapshot to restore from VMID int `json:"vmid,omitempty"` } + +// ForeignKeyArchivesStanza wraps the per-tier list so "evaluated, none" (`tiers: []`) differs from "not evaluated" +// (the stanza absent) without a null on the wire. +type ForeignKeyArchivesStanza struct { + Tiers []ForeignKeyArchives `json:"tiers"` +} + +// ForeignKeyArchives is one tier's count of archives written with another key (R-366 slice 2): the count and the +// newest/oldest archive time (RFC3339, UTC). No key material — the fingerprints stay on the box. +type ForeignKeyArchives struct { + Target string `json:"target"` + Count int `json:"count"` + Oldest string `json:"oldest"` + Newest string `json:"newest"` +}