restore-test: verdict is liveness, not start-task exitstatus (v0.7.0)

Fixes the crying-wolf false-fail surfaced by the live hub-enrollment runbook: PVE's guest-start task exits "WARNINGS: 1" for the benign systemd-nesting advisory, and WaitTask treated any non-OK exitstatus as failure, so the verdict was decided by an advisory exit code before the real boot check ran. Every modern-distro restore-test reported pass:false. - proxmox.WaitOptions.AllowWarnings (opt-in; default keeps all callers strict) - restore-test start step accepts warnings, surfaces them, verdict stays waitRunning - RestoreTestResult.StartWarnings/.WarningsRecognized + version-free "enable nesting" recognizer (can't rot back at systemd 258+); GuestAPI.TaskLogTail - hub.RestoreTest.warnings/.warnings_recognized wire fields (consumed by hub v0.7.5) - scheduler logs clean / passed-with-recognized / passed-with-unrecognized warnings - tests: WaitTask warnings matrix; restore-test pass/fail-on-liveness; version-free regression guard (systemd 256-300) Single agent bump 0.6.0 -> 0.7.0 covering the agent half of both task phases. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 19:30:03 +02:00
parent 7eea638b92
commit 6e86483185
13 changed files with 395 additions and 126 deletions
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"strconv"
 	"testing"
 	"time"

@@ -55,6 +56,136 @@ func TestRunRestoreTest_PassAndTeardown(t *testing.T) {
 	}
 }

+// startWarnAPI builds a fakeAPI whose guest-start task exits "WARNINGS: 1" and whose start
+// task log contains the given warning lines. The guest reaches running (status default).
+func startWarnAPI(startUPID string, logLines []string) *fakeAPI {
+	return &fakeAPI{
+		cfg:       map[int]proxmox.GuestConfig{990000: scratchCfg()},
+		startUPID: startUPID,
+		waitFunc: func(upid string) (proxmox.TaskStatus, error) {
+			if upid == startUPID {
+				return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil
+			}
+			return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
+		},
+		logTailFunc: func(string) ([]string, error) { return logLines, nil },
+	}
+}
+
+func TestRunRestoreTest_PassWithRecognizedWarnings(t *testing.T) {
+	// The crux of the fix: start exits WARNINGS (systemd-nesting advisory) AND the guest
+	// reaches running → PASS. Warnings surfaced, recognized; verdict is liveness, not exit code.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{
+		"run_buffer: starting CT",
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"CT started",
+	})
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+	})
+	if !res.Pass || res.Err != nil {
+		t.Fatalf("start-with-warnings + running must PASS, got %+v", res)
+	}
+	if res.Verified != "boot+running" {
+		t.Errorf("verified = %q", res.Verified)
+	}
+	if len(res.StartWarnings) != 1 || !contains2(res.StartWarnings[0], "enable nesting") {
+		t.Fatalf("the nesting warning must be surfaced, got %+v", res.StartWarnings)
+	}
+	if !res.WarningsRecognized {
+		t.Errorf("the nesting warning must be recognized (benign)")
+	}
+}
+
+func TestRunRestoreTest_PassWithUnrecognizedWarning(t *testing.T) {
+	// An UNRECOGNIZED warning + running still PASSES (verdict is liveness), but is flagged
+	// not-recognized so the operator looks. Visibility-only, never a false-fail.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{"WARN: something unexpected during start"})
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+	})
+	if !res.Pass || res.Err != nil {
+		t.Fatalf("unrecognized warning + running must still PASS, got %+v", res)
+	}
+	if len(res.StartWarnings) != 1 {
+		t.Fatalf("warning must still be surfaced, got %+v", res.StartWarnings)
+	}
+	if res.WarningsRecognized {
+		t.Errorf("an unrecognized warning must NOT be recognized")
+	}
+}
+
+func TestRunRestoreTest_LivenessIsTheVerdict(t *testing.T) {
+	// Start exits WARNINGS but the guest NEVER reaches running → FAIL. The verdict is
+	// liveness; warnings can never turn a non-running guest into a pass.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{"WARN: Systemd 257 detected. You may need to enable nesting."})
+	api.status = map[int]proxmox.Guest{990000: {VMID: 990000, Status: "stopped"}}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+		BootTimeout: 40 * time.Millisecond,
+	})
+	if res.Pass || res.Err == nil {
+		t.Fatalf("not-running must FAIL regardless of warnings, got %+v", res)
+	}
+	if len(api.destroys) != 1 {
+		t.Errorf("teardown must still run: %+v", api.destroys)
+	}
+}
+
+// TestWarningsRecognized_VersionFree is the regression guard: the recognizer must match the
+// nesting advisory for the CURRENT systemd version AND future ones (258/259…), proving the
+// "enable nesting" anchor is version-independent and can't silently rot back into the bug.
+func TestWarningsRecognized_VersionFree(t *testing.T) {
+	for _, v := range []int{256, 257, 258, 259, 300} {
+		line := []string{"WARN: Systemd " + strconv.Itoa(v) + " detected. You may need to enable nesting."}
+		if !warningsRecognized(line) {
+			t.Errorf("systemd %d nesting advisory must be recognized (version-free anchor): %q", v, line[0])
+		}
+	}
+	// Empty ⇒ trivially recognized (N/A).
+	if !warningsRecognized(nil) {
+		t.Error("empty warnings must be trivially recognized")
+	}
+	// An unrelated warning is NOT recognized.
+	if warningsRecognized([]string{"WARN: disk nearly full"}) {
+		t.Error("an unrelated warning must not be recognized")
+	}
+	// Mixed: one benign + one unrelated ⇒ NOT recognized (every line must match).
+	if warningsRecognized([]string{
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"WARN: disk nearly full",
+	}) {
+		t.Error("a mix with any unrecognized line must not be recognized")
+	}
+}
+
+func TestExtractWarningLines(t *testing.T) {
+	got := extractWarningLines([]string{
+		"run_buffer: starting",
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"  WARN: indented warning  ",
+		"INFO: not a warning",
+	})
+	if len(got) != 2 {
+		t.Fatalf("want 2 warning lines, got %d: %+v", len(got), got)
+	}
+	if !contains2(got[0], "enable nesting") || got[1] != "WARN: indented warning" {
+		t.Errorf("warning extraction/trim wrong: %+v", got)
+	}
+}
+
 func TestRunRestoreTest_TeardownOnFailedVerify(t *testing.T) {
 	// Guest never reaches running → verify fails, but teardown MUST still run.
 	api := &fakeAPI{