restore-test: verdict is liveness, not start-task exitstatus (v0.7.0)
Fixes the crying-wolf false-fail surfaced by the live hub-enrollment runbook: PVE's guest-start task exits "WARNINGS: 1" for the benign systemd-nesting advisory, and WaitTask treated any non-OK exitstatus as failure, so the verdict was decided by an advisory exit code before the real boot check ran. Every modern-distro restore-test reported pass:false. - proxmox.WaitOptions.AllowWarnings (opt-in; default keeps all callers strict) - restore-test start step accepts warnings, surfaces them, verdict stays waitRunning - RestoreTestResult.StartWarnings/.WarningsRecognized + version-free "enable nesting" recognizer (can't rot back at systemd 258+); GuestAPI.TaskLogTail - hub.RestoreTest.warnings/.warnings_recognized wire fields (consumed by hub v0.7.5) - scheduler logs clean / passed-with-recognized / passed-with-unrecognized warnings - tests: WaitTask warnings matrix; restore-test pass/fail-on-liveness; version-free regression guard (systemd 256-300) Single agent bump 0.6.0 -> 0.7.0 covering the agent half of both task phases. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -47,6 +47,44 @@ type RestoreTestResult struct {
|
||||
Err error
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
// StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the
|
||||
// systemd-nesting advisory). Populated only when the start exited "WARNINGS: N";
|
||||
// always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning).
|
||||
StartWarnings []string
|
||||
// WarningsRecognized is true iff every StartWarnings line matches the benign anchor.
|
||||
// It affects VISIBILITY ONLY (log level / operator attention), never the verdict — so a
|
||||
// wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and
|
||||
// never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A).
|
||||
WarningsRecognized bool
|
||||
}
|
||||
|
||||
// benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start
|
||||
// advisory ("Systemd <N> detected. You may need to enable nesting."). It carries no systemd
|
||||
// version number, so — unlike an exact-string allowlist on "Systemd 257…" — it cannot rot back
|
||||
// into the false-fail bug as guests move to systemd 258+. Matched case-insensitively.
|
||||
const benignWarningAnchor = "enable nesting"
|
||||
|
||||
// extractWarningLines pulls the warning lines out of a task log tail. PVE prefixes task
|
||||
// warnings with "WARN" (e.g. "WARN: Systemd 257 detected…"); we keep those, trimmed.
|
||||
func extractWarningLines(logTail []string) []string {
|
||||
var out []string
|
||||
for _, l := range logTail {
|
||||
if t := strings.TrimSpace(l); strings.HasPrefix(t, "WARN") {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// warningsRecognized reports whether EVERY warning line is the benign anchor. Empty ⇒ true
|
||||
// (no warnings to worry about). One unrecognized line ⇒ false (operator should look).
|
||||
func warningsRecognized(warnings []string) bool {
|
||||
for _, w := range warnings {
|
||||
if !strings.Contains(strings.ToLower(w), benignWarningAnchor) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch
|
||||
@@ -152,16 +190,33 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
|
||||
}
|
||||
|
||||
// 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8).
|
||||
// The VERDICT is liveness (waitRunning), NEVER the start task's exitstatus. A start
|
||||
// that completes with warnings (e.g. the systemd-nesting advisory → exit "WARNINGS: N")
|
||||
// and then reaches running is a PASS — deciding pass/fail on an advisory exit code is
|
||||
// the crying-wolf bug this guards against. We pass AllowWarnings so WaitTask doesn't
|
||||
// hard-fail on it, then fetch + surface the warning text (visibility only).
|
||||
startUPID, err := e.api.Start(ctx, vmid)
|
||||
if err != nil {
|
||||
res.Err = fmt.Errorf("reconcile: restore-test start: %w", err)
|
||||
return
|
||||
}
|
||||
if startUPID != "" {
|
||||
if _, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{}); err != nil {
|
||||
st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
|
||||
if err != nil {
|
||||
// A real (non-WARNINGS) start-task failure still fails the test.
|
||||
res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
|
||||
// Surface the warning(s); do NOT fail. Liveness below is the verdict.
|
||||
tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
|
||||
if logErr != nil {
|
||||
e.logger.Warn("restore-test: could not read start-task log for warnings",
|
||||
"vmid", vmid, "err", logErr)
|
||||
}
|
||||
res.StartWarnings = extractWarningLines(tail)
|
||||
res.WarningsRecognized = warningsRecognized(res.StartWarnings)
|
||||
}
|
||||
}
|
||||
if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil {
|
||||
res.Err = err
|
||||
|
||||
Reference in New Issue
Block a user