R-86: restore-test follows the backup, not the clock (v0.121.0)

The ticker survives as the EVALUATION interval only. A tier is DUE when its
newest archive that has settled for `settle` (default 24h) has not been proven:
daily tier -> proved daily on yesterday's archive, weekly tier -> weekly on its
own, newborn -> UNKNOWN.

The trap avoided: the literal reading ("newest archive is >= 24h old") is NEVER
true on a daily tier, so it silently switches restore-testing off where it
matters most. Red-proved at 0 runs over 5 simulated days.

- state records WHICH archive was proven; legacy files keep their time and yield
  no proven archive (each tier due once after the upgrade, deliberately)
- two knobs replace one: restore_test_eval_interval_seconds (6h, measured) and
  restore_test_settle_seconds (24h). The old cadence key keeps its DISABLE
  meaning verbatim and now seeds the settle lag, with a start-up WARN.
- due-check runs BEFORE the heavy-op gate (a frequent poll must not make a
  starting backup record a failure, F-A1)
- candidate picker skips implausible archives (a phantom would be due forever)
- new read-only --selftest=restore-test-due prints the verdict + its cost

cmd/felhom-agent/main.go

@@ -165,7 +165,7 @@ func main() {
 		showVersion   bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-cores/-memory (-sysdata-grow is deprecated: folded into -datavol-grow); keeps the guest)")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `restore-test-due` = READ-ONLY: print the per-tier due verdict the scheduler would act on, with its cost; `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-cores/-memory (-sysdata-grow is deprecated: folded into -datavol-grow); keeps the guest)")
 	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up")
 	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
 	flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)")
@@ -238,6 +238,8 @@ func main() {
 		os.Exit(runSelftestBackup(context.Background(), cfg, logger, vmid))
 	case "restore-test":
 		os.Exit(runSelftestRestoreTest(context.Background(), cfg, logger, archive))
+	case "restore-test-due":
+		os.Exit(runSelftestRestoreTestDue(context.Background(), cfg, logger))
 	case "pbs-verify":
 		os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
 	case "lanresolver":
@@ -1273,13 +1275,21 @@ func primaryBackupTargetOf(cfg config.Config) func() hub.ConfiguredBackupTarget
 // scratch band / restore storage is invalid — a misconfig must not crash the daemon, and the
 // machinery still works on-demand via --selftest=restore-test.
 func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *reconcile.Engine, store *backup.Store, rtState *backup.RestoreTestState, inFlight *backup.InFlight, logger *slog.Logger) *backup.Scheduler {
-	cadence := cfg.Backup.RestoreTestCadence()
+	// R-86: this is the EVALUATION interval, not the trigger. What decides a test happens is the
+	// per-archive due-check in internal/backup/restoretest_due.go.
+	cadence := cfg.Backup.RestoreTestEvalInterval()
 	if cadence > 0 {
 		if err := cfg.Backup.ValidateForRestoreTest(); err != nil {
-			logger.Warn("daemon: restore-test cadence disabled (config invalid)", "err", err)
+			logger.Warn("daemon: restore-test disabled (config invalid)", "err", err)
 			cadence = 0
 		}
 	}
+	if cadence > 0 && cfg.Backup.RestoreTestLegacyCadenceInUse() {
+		// Said ONCE, at start-up, naming both replacements: a key whose meaning changed under a box
+		// without a word is the silent repurposing R-86 §8.3 forbids.
+		logger.Warn("daemon: backup.restore_test_cadence_seconds is DEPRECATED — R-86 replaced the interval trigger with a per-archive due-check; this value now seeds the SETTLE lag only. Set backup.restore_test_settle_seconds and backup.restore_test_eval_interval_seconds explicitly",
+			"settle", cfg.Backup.RestoreTestSettle(), "eval_interval", cadence)
+	}
 	min, max := cfg.Backup.ScratchBand()
 	target := cfg.Backup.BackupTarget()
 	runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", "", logger)
@@ -1315,13 +1325,18 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
 			}
 		},
 		Cadence: cadence,
-		Logger:  logger,
+		// R-86: the settle lag — how long an archive must have sat before it is a candidate. With
+		// the per-archive due-check, this plus the archive rhythm is the whole schedule.
+		Settle: cfg.Backup.RestoreTestSettle(),
+		Logger: logger,
 
 		// R-85: rotate across EVERY configured tier, oldest-proven first (operator ruling, Option 1).
 		// Before this the scheduler only ever saw cfg.Backup.BackupTarget(), so the offsite tier's
 		// archives were never candidates and the DR tier went unproven for its whole existence.
+		// R-86 demoted that ordering to the tie-break BETWEEN DUE TIERS and widened this picker to
+		// the settle-aware one, which is what makes due-ness per archive generation.
 		Tiers:    tierIDs,
-		TierPick: runner.PickRestoreCandidateOn,
+		TierPick: runner.PickSettledRestoreCandidateOn,
 		State:    rtState,
 		InFlight: inFlight,
 	})
@@ -1751,6 +1766,67 @@ func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logg
 // running → teardown) of -archive (or the newest backup on the local target) into a scratch
 // guest. Standalone (no hub). Runs engine.Recover first so a leaked scratch from a prior
 // crashed test is reaped before this run.
+// runSelftestRestoreTestDue prints the per-tier DUE verdict the scheduler would act on, and what
+// each evaluation COST — read-only, so it is safe on any box at any time.
+//
+// It exists for two reasons R-86 needed and could not get from a log line. First, the due-check's
+// verdict is the whole schedule now: "why did nothing run last night?" is answerable only by asking
+// the same question the scheduler asks, against the same storages, in the same order. Second, the
+// evaluation interval had to be chosen from a MEASURED cost rather than a guess — an offsite tier's
+// candidate lookup crosses the WAN, and a monitoring loop that costs more than it is worth is how a
+// check becomes the load. It reuses the daemon's own construction path (buildRestoreTestScheduler),
+// so what it prints is what the daemon would decide, not a re-derivation of it.
+func runSelftestRestoreTestDue(ctx context.Context, cfg config.Config, logger *slog.Logger) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	rtState := backup.NewRestoreTestState(filepath.Join(cfg.OOB.WithDefaults().StateDir, "restore-test-state.json"))
+	sched := buildRestoreTestScheduler(cfg, px, nil, backup.NewStore(), rtState, &backup.InFlight{}, logger)
+
+	fmt.Printf("eval_interval=%s settle=%s\n", cfg.Backup.RestoreTestEvalInterval(), cfg.Backup.RestoreTestSettle())
+	start := time.Now()
+	verdicts := sched.EvaluateDue(ctx)
+	total := time.Since(start)
+	if len(verdicts) == 0 {
+		fmt.Println("no tiers configured for restore-testing (or rotation not wired)")
+		return 0
+	}
+	rc := 0
+	for _, v := range verdicts {
+		proven, _ := rtState.ProvenArchive(v.Target)
+		fmt.Printf("tier=%-16s due=%-5v archive=%q landed=%s proven=%q\n  reason: %s\n",
+			v.Target, v.Due, v.Archive, formatOrDash(v.Landed), proven, v.Reason)
+		if v.Err != nil {
+			// A tier we could not list is UNKNOWN, and it is a non-zero exit: an unreadable tier is
+			// a real condition, not a quiet "nothing to do".
+			fmt.Printf("  ERROR: %v\n", v.Err)
+			rc = 3
+		}
+	}
+	// Per-tier timing, measured one tier at a time so the WAN leg is attributable (R-86 Part 1.4).
+	for _, v := range verdicts {
+		t0 := time.Now()
+		_ = sched.EvaluateDueTier(ctx, v.Target)
+		fmt.Printf("cost tier=%-16s one_lookup=%s\n", v.Target, time.Since(t0).Round(time.Millisecond))
+	}
+	fmt.Printf("cost all_tiers=%s\n", total.Round(time.Millisecond))
+	return rc
+}
+
+// formatOrDash renders a time, or "-" when it is zero (no archive).
+func formatOrDash(t time.Time) string {
+	if t.IsZero() {
+		return "-"
+	}
+	return t.UTC().Format(time.RFC3339)
+}
+
 func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog.Logger, archive string) int {
 	if err := cfg.Validate(); err != nil {
 		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
@@ -2940,6 +3016,8 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "backup"
 	case "restore-test":
 		f.mode = "restore-test"
+	case "restore-test-due":
+		f.mode = "restore-test-due"
 	case "pbs-verify":
 		f.mode = "pbs-verify"
 	case "lanresolver":
@@ -2957,7 +3035,7 @@ func (f *selftestFlag) Set(v string) error {
 	case "controller-swap":
 		f.mode = "controller-swap"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|pbs-verify|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap)", v)
 	}
 	return nil
 }

cmd/felhom-agent/restoretest_due_wiring_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+)
+
+// R-86 Scenario I — the seam-discipline test for the due-check.
+//
+// A due-check is worth nothing if the daemon still wires the OLD picker: every unit test in
+// internal/backup would stay green (they inject the seam directly), the scheduler would ask for the
+// newest archive with no settle cutoff, and the per-archive rule would run against a candidate that
+// changes every time a backup lands. That is the same shape as the v0.91.0 inert seam — built,
+// tested, never called — and this repo has shipped it four times.
+//
+// It walks main.go's AST rather than grepping: a commented-out call still satisfies a substring
+// match, and a comment is not a caller.
+func TestMainWiresTheSettleAwareTierPicker(t *testing.T) {
+	f := parseMainForWiring(t)
+
+	var settlePicker, oldPicker, settleWired, evalInterval bool
+	ast.Inspect(f, func(n ast.Node) bool {
+		switch node := n.(type) {
+		case *ast.SelectorExpr:
+			// runner.PickSettledRestoreCandidateOn passed as a value (not called).
+			switch node.Sel.Name {
+			case "PickSettledRestoreCandidateOn":
+				settlePicker = true
+			case "PickRestoreCandidateOn":
+				oldPicker = true
+			}
+		case *ast.KeyValueExpr:
+			key, ok := node.Key.(*ast.Ident)
+			if !ok {
+				return true
+			}
+			if key.Name == "Settle" {
+				settleWired = true
+			}
+		case *ast.CallExpr:
+			if sel, ok := node.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "RestoreTestEvalInterval" {
+				evalInterval = true
+			}
+		}
+		return true
+	})
+
+	if !settlePicker {
+		t.Error("main.go never passes runner.PickSettledRestoreCandidateOn as the scheduler's TierPick — " +
+			"the due-check would run without a settle cutoff, i.e. against an archive that may still be being written")
+	}
+	if oldPicker {
+		t.Error("main.go still wires the pre-R-86 PickRestoreCandidateOn as a tier picker — " +
+			"two pickers means the one under test is not the one running")
+	}
+	if !settleWired {
+		t.Error("main.go never sets SchedulerOptions.Settle — the settle lag would default to 0 in the daemon " +
+			"and every freshly-landed archive would be an immediate candidate")
+	}
+	if !evalInterval {
+		t.Error("main.go never calls cfg.Backup.RestoreTestEvalInterval() — the scheduler would be driven by " +
+			"the retired cadence knob")
+	}
+}
+
+// The two R-85 guarantees the due-check must not have quietly dropped: the spec is still built PER
+// RUN, and the shared heavy-operation gate is still handed to the scheduler.
+func TestMainStillWiresTheHeavyOperationGateAndPerRunSpec(t *testing.T) {
+	f := parseMainForWiring(t)
+
+	var inFlightWired, specIsAFunc bool
+	ast.Inspect(f, func(n ast.Node) bool {
+		kv, ok := n.(*ast.KeyValueExpr)
+		if !ok {
+			return true
+		}
+		key, ok := kv.Key.(*ast.Ident)
+		if !ok {
+			return true
+		}
+		switch key.Name {
+		case "InFlight":
+			inFlightWired = true
+		case "Spec":
+			// A FuncLit means it is evaluated per run; anything else is a frozen value.
+			if _, isFunc := kv.Value.(*ast.FuncLit); isFunc {
+				specIsAFunc = true
+			}
+		}
+		return true
+	})
+
+	if !inFlightWired {
+		t.Error("main.go no longer hands the scheduler the shared InFlight gate — a restore-test could pull a " +
+			"multi-GB archive over the same tunnel an offsite backup is pushing one over (Scenario F)")
+	}
+	if !specIsAFunc {
+		t.Error("SchedulerOptions.Spec is no longer a function literal — a frozen spec is the R-85 defect " +
+			"(the tier and its timeout evaluated once at daemon start, forever)")
+	}
+}
+
+func parseMainForWiring(t *testing.T) *ast.File {
+	t.Helper()
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "main.go", nil, 0)
+	if err != nil {
+		t.Fatalf("parse main.go: %v", err)
+	}
+	return f
+}