admin/felhom-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

R-85 Phase 2: tier rotation, persisted state, one heavy op at a time

Browse Source 
The scheduler could only ever see cfg.Backup.BackupTarget(), so the offsite
tier's archives were never candidates — which is why demo-hp's DR tier reported
'applied' with zero snapshots for five days and nobody noticed.

Selection: oldest-first (operator ruling, Option 1). Never-proven sorts first,
which is where the offsite tier starts. Ties break on target id so ordering is
deterministic rather than following Go's randomised map order. Rotation credit
only on SUCCESS — a permanently failing tier must keep sorting first, not look
freshly proven and stop being retried.

- backup.RestoreTestState: persisted last-success per tier (atomic tmp+rename).
  This genuinely needs persistence unlike R-84: R-84 had ground truth to consult
  (the archive is still on the storage), whereas a restore-test destroys its
  scratch and leaves no artifact. Corrupt/missing file -> 'nothing proven'.
- backup.InFlight: host-wide one-heavy-op gate shared with the local-API backup
  path. A LINK concern, not a lock one — an offsite restore pulls multi-GB over
  the same tunnel a backup pushes one, and at ~33 MB/min both drift toward
  timeout, which is how a healthy tier gets recorded as failed. Callers DEFER,
  never cancel.
- PickRestoreCandidateOn: newest archive on a named tier; '' is not an error, or
  every fresh box looks broken for its first week.
- An empty tier is skipped and the next tried; it cannot starve, since it is
  still least-recently-proven once it has an archive.
- POST /backup joins the gate (409 naming the holder).

Red-proofs A/E/F observed with the documented text. Full suite green (29
packages, rc=0).
This commit is contained in:
Claude Code
2026-07-26 21:00:42 +02:00
parent 765d8b3168
commit 043c7622bc
8 changed files with 743 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/felhom-agent/main.go
+25 -6
View File
@@ -649,7 +649,11 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
// Self-restore-test scheduler (slice 6): the fourth daemon goroutine. Runs the restore-
// test on the configured cadence (default 24h). Disabled cleanly when the cadence is off
// OR the scratch band / restore storage is misconfigured — the daemon still runs.
scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, logger)
// R-85: persisted per-tier restore-test state + the host-wide one-heavy-op gate, both shared
// with the local API so a backup and a restore-test can never run together.
rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
heavyOps := &backup.InFlight{}
scheduler := buildRestoreTestScheduler(cfg, px, engine, backupStore, rtState, heavyOps, logger)
// PBS verify loop (slice 6 Phase B): the fifth daemon goroutine. Cheap, key-free,
// ciphertext-level integrity check on its own cadence (default 6h), reporting per-snapshot
@@ -757,7 +761,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
return false
},
}
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
if localTokens != nil {
defer localTokens.Close()
}
@@ -1232,7 +1236,7 @@ func readTrimmed(path string) (string, error) {
// disables the cadence (returns a scheduler that just waits) when the cadence is off or the
// scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
// machinery still works on-demand via --selftest=restore-test.
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, logger *slog.Logger) *backup.Scheduler {
func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, logger *slog.Logger) *backup.Scheduler {
cadence := cfg.Backup.RestoreTestCadence()
if cadence > 0 {
if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
@@ -1243,6 +1247,12 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
min, max := cfg.Backup.ScratchBand()
target := cfg.Backup.BackupTarget()
runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
// Every configured tier is a rotation candidate, not just the primary.
cfgTiers, _ := cfg.Backup.BackupTiers() // warnings already logged where the tiers are armed
tierIDs := make([]string, 0, len(cfgTiers))
for _, t := range cfgTiers {
tierIDs = append(tierIDs, t.TargetID)
}
return backup.NewScheduler(backup.SchedulerOptions{
Runner: engine,
Pick: runner.PickRestoreCandidate,
@@ -1270,6 +1280,14 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
},
Cadence: cadence,
Logger: logger,
// R-85: rotate across EVERY configured tier, oldest-proven first (operator ruling, Option 1).
// Before this the scheduler only ever saw cfg.Backup.BackupTarget(), so the offsite tier's
// archives were never candidates and the DR tier went unproven for its whole existence.
Tiers: tierIDs,
TierPick: runner.PickRestoreCandidateOn,
State: rtState,
InFlight: inFlight,
})
}
@@ -1278,7 +1296,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
// fixed. The opened token store is returned via outTokens so the caller can Close it.
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, inFlight *backup.InFlight, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, escrowCeremony *localapi.EscrowCeremonyConfig, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
if !cfg.LocalAPI.Enabled() {
return nil
}
@@ -1357,6 +1375,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
Guests: px,
Backups: runner,
BackupTiers: apiTiers, // R-82: primary first; untargeted endpoints act on the primary
InFlight: inFlight, // R-85: shared with the restore-test scheduler (Scenario F)
Store: store,
Storage: observer,
DriveTargets: driveTargets, // Impl-2a: registry+units drives for the /disks view (union w/ Observe storages)
@@ -1846,7 +1865,7 @@ func runSelftestBringUp(ctx context.Context, cfg config.Config, logger *slog.Log
Cores: sizing.Cores, MemoryMB: sizing.MemoryMB,
RootfsGrowGB: sizing.RootfsGrowGB, DataVolGrowGB: sizing.DataVolGrowGB, DataVolMount: sizing.DataVolMount,
SysDataGrowGB: sizing.SysDataGrowGB, SysDataMount: sizing.SysDataMount,
IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
}
fmt.Printf(" bringing up %s → vmid %d on %s …\n", archive, vmid, cfg.Backup.RestoreStorage)
res := engine.RunBringUp(ctx, spec)
@@ -2010,7 +2029,7 @@ func runSelftestProvision(ctx context.Context, cfg config.Config, logger *slog.L
Cores: a.sizing.Cores, MemoryMB: a.sizing.MemoryMB,
RootfsGrowGB: a.sizing.RootfsGrowGB, DataVolGrowGB: a.sizing.DataVolGrowGB, DataVolMount: a.sizing.DataVolMount,
SysDataGrowGB: a.sizing.SysDataGrowGB, SysDataMount: a.sizing.SysDataMount,
IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
IslandBridge: cfg.LocalAPI.IslandBridge, IslandGuestAddr: cfg.LocalAPI.IslandGuestAddr, // R-50 island NIC (both empty = pre-R-50)
})
if res.Err != nil || !res.Pass {
fmt.Fprintf(os.Stderr, " [FAIL] front-half bring-up (vmid %d): %v\n", a.vmid, res.Err)