restore-test: verdict is liveness, not start-task exitstatus (v0.7.0)

Fixes the crying-wolf false-fail surfaced by the live hub-enrollment runbook:
PVE's guest-start task exits "WARNINGS: 1" for the benign systemd-nesting
advisory, and WaitTask treated any non-OK exitstatus as failure, so the verdict
was decided by an advisory exit code before the real boot check ran. Every
modern-distro restore-test reported pass:false.

- proxmox.WaitOptions.AllowWarnings (opt-in; default keeps all callers strict)
- restore-test start step accepts warnings, surfaces them, verdict stays waitRunning
- RestoreTestResult.StartWarnings/.WarningsRecognized + version-free "enable
  nesting" recognizer (can't rot back at systemd 258+); GuestAPI.TaskLogTail
- hub.RestoreTest.warnings/.warnings_recognized wire fields (consumed by hub v0.7.5)
- scheduler logs clean / passed-with-recognized / passed-with-unrecognized warnings
- tests: WaitTask warnings matrix; restore-test pass/fail-on-liveness; version-free
  regression guard (systemd 256-300)

Single agent bump 0.6.0 -> 0.7.0 covering the agent half of both task phases.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 19:30:03 +02:00
parent 7eea638b92
commit 6e86483185
13 changed files with 395 additions and 126 deletions
+56 -1
View File
@@ -47,6 +47,44 @@ type RestoreTestResult struct {
Err error
StartedAt time.Time
Duration time.Duration
// StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the
// systemd-nesting advisory). Populated only when the start exited "WARNINGS: N";
// always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning).
StartWarnings []string
// WarningsRecognized is true iff every StartWarnings line matches the benign anchor.
// It affects VISIBILITY ONLY (log level / operator attention), never the verdict — so a
// wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and
// never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A).
WarningsRecognized bool
}
// benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start
// advisory ("Systemd <N> detected. You may need to enable nesting."). It carries no systemd
// version number, so — unlike an exact-string allowlist on "Systemd 257…" — it cannot rot back
// into the false-fail bug as guests move to systemd 258+. Matched case-insensitively.
const benignWarningAnchor = "enable nesting"
// extractWarningLines pulls the warning lines out of a task log tail. PVE prefixes task
// warnings with "WARN" (e.g. "WARN: Systemd 257 detected…"); we keep those, trimmed.
func extractWarningLines(logTail []string) []string {
var out []string
for _, l := range logTail {
if t := strings.TrimSpace(l); strings.HasPrefix(t, "WARN") {
out = append(out, t)
}
}
return out
}
// warningsRecognized reports whether EVERY warning line is the benign anchor. Empty ⇒ true
// (no warnings to worry about). One unrecognized line ⇒ false (operator should look).
func warningsRecognized(warnings []string) bool {
for _, w := range warnings {
if !strings.Contains(strings.ToLower(w), benignWarningAnchor) {
return false
}
}
return true
}
// IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch
@@ -152,16 +190,33 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
}
// 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8).
// The VERDICT is liveness (waitRunning), NEVER the start task's exitstatus. A start
// that completes with warnings (e.g. the systemd-nesting advisory → exit "WARNINGS: N")
// and then reaches running is a PASS — deciding pass/fail on an advisory exit code is
// the crying-wolf bug this guards against. We pass AllowWarnings so WaitTask doesn't
// hard-fail on it, then fetch + surface the warning text (visibility only).
startUPID, err := e.api.Start(ctx, vmid)
if err != nil {
res.Err = fmt.Errorf("reconcile: restore-test start: %w", err)
return
}
if startUPID != "" {
if _, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{}); err != nil {
st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
if err != nil {
// A real (non-WARNINGS) start-task failure still fails the test.
res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err)
return
}
if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
// Surface the warning(s); do NOT fail. Liveness below is the verdict.
tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
if logErr != nil {
e.logger.Warn("restore-test: could not read start-task log for warnings",
"vmid", vmid, "err", logErr)
}
res.StartWarnings = extractWarningLines(tail)
res.WarningsRecognized = warningsRecognized(res.StartWarnings)
}
}
if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil {
res.Err = err