restore-test: verdict is liveness, not start-task exitstatus (v0.7.0)

Fixes the crying-wolf false-fail surfaced by the live hub-enrollment runbook:
PVE's guest-start task exits "WARNINGS: 1" for the benign systemd-nesting
advisory, and WaitTask treated any non-OK exitstatus as failure, so the verdict
was decided by an advisory exit code before the real boot check ran. Every
modern-distro restore-test reported pass:false.

- proxmox.WaitOptions.AllowWarnings (opt-in; default keeps all callers strict)
- restore-test start step accepts warnings, surfaces them, verdict stays waitRunning
- RestoreTestResult.StartWarnings/.WarningsRecognized + version-free "enable
  nesting" recognizer (can't rot back at systemd 258+); GuestAPI.TaskLogTail
- hub.RestoreTest.warnings/.warnings_recognized wire fields (consumed by hub v0.7.5)
- scheduler logs clean / passed-with-recognized / passed-with-unrecognized warnings
- tests: WaitTask warnings matrix; restore-test pass/fail-on-liveness; version-free
  regression guard (systemd 256-300)

Single agent bump 0.6.0 -> 0.7.0 covering the agent half of both task phases.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

internal/backup/runner.go

@@ -205,13 +205,15 @@ func mountpointLabel(key, cfg string) string {
 // package owns the reconcile→hub mapping so reconcile need not import hub for the result).
 func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.RestoreTest {
 	rt := hub.RestoreTest{
-		SourceArchive:   res.Archive,
-		SourceTier:      res.SourceTier,
-		ScratchVMID:     res.ScratchVMID,
-		Pass:            res.Pass,
-		Verified:        res.Verified,
-		TestedAt:        testedAt.Format(time.RFC3339),
-		DurationSeconds: res.Duration.Seconds(),
+		SourceArchive:      res.Archive,
+		SourceTier:         res.SourceTier,
+		ScratchVMID:        res.ScratchVMID,
+		Pass:               res.Pass,
+		Verified:           res.Verified,
+		TestedAt:           testedAt.Format(time.RFC3339),
+		DurationSeconds:    res.Duration.Seconds(),
+		Warnings:           res.StartWarnings,
+		WarningsRecognized: res.WarningsRecognized,
 	}
 	if res.Err != nil {
 		rt.Error = res.Err.Error()

internal/backup/schedule.go

@@ -102,10 +102,19 @@ func (s *Scheduler) tick(ctx context.Context) {
 	}
 	rt := ToHubRestoreTest(res, s.now())
 	s.store.RecordRestoreTest(rt)
-	if rt.Pass {
-		s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
-	} else {
+	switch {
+	case !rt.Pass:
 		// A failing restore-test is the loudest DR signal there is.
 		s.logger.Error("backup: scheduled restore-test FAILED", "archive", rt.SourceArchive, "err", rt.Error)
+	case len(res.StartWarnings) == 0:
+		s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
+	case res.WarningsRecognized:
+		// Passed; the only warnings are the known-benign (e.g. systemd-nesting) advisory.
+		s.logger.Info("backup: scheduled restore-test passed with warnings (recognized)",
+			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
+	default:
+		// Passed liveness, but an UNRECOGNIZED start warning stood out — worth an operator look.
+		s.logger.Warn("backup: scheduled restore-test passed with UNRECOGNIZED warnings",
+			"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
 	}
 }

internal/hub/report.go

@@ -196,6 +196,14 @@ type RestoreTest struct {
 	Error           string  `json:"error,omitempty"`
 	TestedAt        string  `json:"tested_at"` // RFC3339
 	DurationSeconds float64 `json:"duration_seconds"`
+	// Warnings are the guest-start task's warning line(s) (e.g. the systemd-nesting advisory).
+	// Present on a PASS that emitted warnings; pass/fail itself is liveness-only, so a passed
+	// restore-test can carry warnings. Omitted when there are none.
+	Warnings []string `json:"warnings,omitempty"`
+	// WarningsRecognized is true iff every Warnings line is the known-benign anchor. Omitted
+	// (⇒ false) when absent — and false is the SAFE default: the hub then treats it as an
+	// unrecognized warning (louder), so a missing flag can only over-notice, never hide.
+	WarningsRecognized bool `json:"warnings_recognized,omitempty"`
 }
 
 // PBSSnapshot is one PBS (offsite) snapshot's inventory + integrity state (doc 03 §8, slice

internal/proxmox/task.go

@@ -3,6 +3,7 @@ package proxmox
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 )
 
@@ -42,6 +43,14 @@ type WaitOptions struct {
 	// Timeout bounds the whole wait (default 10m). Restore/vzdump can be slow;
 	// callers may raise it. A zero/elapsed context deadline also stops the wait.
 	Timeout time.Duration
+	// AllowWarnings accepts a task that completes with a "WARNINGS: N" exitstatus as
+	// success (returning the TaskStatus with ExitStatus intact, nil error), instead of
+	// the default hard failure. Opt-in PER CALL — the default (false) keeps every existing
+	// caller strict, because vzdump/restore/destroy warnings can be meaningful. Only the
+	// restore-test's guest-start step opts in (a start advisory like the systemd-nesting
+	// notice must not false-fail a guest that actually boots; doc 03 §8). A non-WARNINGS
+	// non-OK exit is still a *TaskError regardless of this flag.
+	AllowWarnings bool
 }
 
 func (o WaitOptions) withDefaults() WaitOptions {
@@ -135,6 +144,12 @@ func (c *Client) WaitTask(ctx context.Context, upid string, opts WaitOptions) (T
 		if st.ExitStatus == "OK" {
 			return st, nil
 		}
+		// Warnings-accepted path (opt-in): a "WARNINGS: N" exit is returned as success with
+		// ExitStatus intact, so the caller can fetch/surface the warning text from the log
+		// without the test/op failing on an advisory. Any other non-OK exit still fails.
+		if opts.AllowWarnings && strings.HasPrefix(st.ExitStatus, "WARNINGS") {
+			return st, nil
+		}
 		tail, _ := c.TaskLogTail(ctx, upid, 20) // best-effort
 		return st, newTaskError(upid, st.ExitStatus, tail)
 	}

internal/proxmox/task_test.go

@@ -56,6 +56,55 @@ func TestWaitTask_FailedSurfacesPrivilege(t *testing.T) {
 	}
 }
 
+func TestWaitTask_AllowWarnings_Accepts(t *testing.T) {
+	// A start task that completes with the systemd-nesting advisory exits "WARNINGS: 1".
+	// With AllowWarnings, that's success and ExitStatus is returned intact for the caller.
+	d := &mockDoer{fn: func(r *http.Request) (*http.Response, error) {
+		return jsonResp(200, `{"data":{"upid":"`+testUPID+`","status":"stopped","exitstatus":"WARNINGS: 1"}}`), nil
+	}}
+	opts := fastWait
+	opts.AllowWarnings = true
+	st, err := newTestClient(d).WaitTask(context.Background(), testUPID, opts)
+	if err != nil {
+		t.Fatalf("AllowWarnings should accept WARNINGS: %v", err)
+	}
+	if st.ExitStatus != "WARNINGS: 1" {
+		t.Errorf("ExitStatus = %q, want %q (must be returned intact so the caller can read it)", st.ExitStatus, "WARNINGS: 1")
+	}
+}
+
+func TestWaitTask_AllowWarnings_RealErrorStillFails(t *testing.T) {
+	// AllowWarnings must NOT swallow a genuine non-WARNINGS failure.
+	d := &mockDoer{fn: func(r *http.Request) (*http.Response, error) {
+		if strings.Contains(r.URL.Path, "/log") {
+			return jsonResp(200, `{"data":[{"n":1,"t":"TASK ERROR: 403 Permission check failed (/vms/9000, VM.PowerMgmt)"}]}`), nil
+		}
+		return jsonResp(200, `{"data":{"upid":"`+testUPID+`","status":"stopped","exitstatus":"403 Permission check failed (/vms/9000, VM.PowerMgmt)"}}`), nil
+	}}
+	opts := fastWait
+	opts.AllowWarnings = true
+	_, err := newTestClient(d).WaitTask(context.Background(), testUPID, opts)
+	var te *TaskError
+	if !errors.As(err, &te) {
+		t.Fatalf("a real error must still be *TaskError even with AllowWarnings, got %T: %v", err, err)
+	}
+}
+
+func TestWaitTask_DefaultRejectsWarnings(t *testing.T) {
+	// Default (AllowWarnings:false) keeps every existing caller strict: WARNINGS → *TaskError.
+	d := &mockDoer{fn: func(r *http.Request) (*http.Response, error) {
+		if strings.Contains(r.URL.Path, "/log") {
+			return jsonResp(200, `{"data":[{"n":1,"t":"WARN: Systemd 257 detected. You may need to enable nesting."}]}`), nil
+		}
+		return jsonResp(200, `{"data":{"upid":"`+testUPID+`","status":"stopped","exitstatus":"WARNINGS: 1"}}`), nil
+	}}
+	_, err := newTestClient(d).WaitTask(context.Background(), testUPID, fastWait) // AllowWarnings:false
+	var te *TaskError
+	if !errors.As(err, &te) {
+		t.Fatalf("default must still fail on WARNINGS (existing callers unaffected), got %T: %v", err, err)
+	}
+}
+
 func TestWaitTask_Timeout(t *testing.T) {
 	d := &mockDoer{fn: func(r *http.Request) (*http.Response, error) {
 		return jsonResp(200, `{"data":{"upid":"`+testUPID+`","status":"running"}}`), nil

internal/reconcile/engine_test.go

@@ -29,6 +29,8 @@ type fakeAPI struct {
 	waitFunc func(upid string) (proxmox.TaskStatus, error)
 	// statusFunc backs TaskStatusOnce (crash recovery); default = stopped/OK.
 	statusFunc func(upid string) (proxmox.TaskStatus, error)
+	// logTailFunc backs TaskLogTail (restore-test start-warning surfacing); default = empty.
+	logTailFunc func(upid string) ([]string, error)
 
 	starts   []int
 	stops    []int
@@ -75,6 +77,13 @@ func (f *fakeAPI) TaskStatusOnce(_ context.Context, upid string) (proxmox.TaskSt
 	return proxmox.TaskStatus{UPID: upid, Status: "stopped", ExitStatus: "OK"}, nil
 }
 
+func (f *fakeAPI) TaskLogTail(_ context.Context, upid string, _ int) ([]string, error) {
+	if f.logTailFunc != nil {
+		return f.logTailFunc(upid)
+	}
+	return nil, nil
+}
+
 type setCall struct {
 	vmid   int
 	params map[string]string

internal/reconcile/restoretest.go

@@ -47,6 +47,44 @@ type RestoreTestResult struct {
 	Err         error
 	StartedAt   time.Time
 	Duration    time.Duration
+	// StartWarnings holds the warning line(s) the guest-start task emitted (e.g. the
+	// systemd-nesting advisory). Populated only when the start exited "WARNINGS: N";
+	// always surfaced, NEVER used to decide pass/fail (the verdict is liveness — waitRunning).
+	StartWarnings []string
+	// WarningsRecognized is true iff every StartWarnings line matches the benign anchor.
+	// It affects VISIBILITY ONLY (log level / operator attention), never the verdict — so a
+	// wrong/stale recognizer can at worst over-notice a benign warning, never false-fail and
+	// never hide a real one. Empty StartWarnings ⇒ trivially recognized (N/A).
+	WarningsRecognized bool
+}
+
+// benignWarningAnchor is a deliberately version-FREE substring of the systemd-nesting start
+// advisory ("Systemd <N> detected. You may need to enable nesting."). It carries no systemd
+// version number, so — unlike an exact-string allowlist on "Systemd 257…" — it cannot rot back
+// into the false-fail bug as guests move to systemd 258+. Matched case-insensitively.
+const benignWarningAnchor = "enable nesting"
+
+// extractWarningLines pulls the warning lines out of a task log tail. PVE prefixes task
+// warnings with "WARN" (e.g. "WARN: Systemd 257 detected…"); we keep those, trimmed.
+func extractWarningLines(logTail []string) []string {
+	var out []string
+	for _, l := range logTail {
+		if t := strings.TrimSpace(l); strings.HasPrefix(t, "WARN") {
+			out = append(out, t)
+		}
+	}
+	return out
+}
+
+// warningsRecognized reports whether EVERY warning line is the benign anchor. Empty ⇒ true
+// (no warnings to worry about). One unrecognized line ⇒ false (operator should look).
+func warningsRecognized(warnings []string) bool {
+	for _, w := range warnings {
+		if !strings.Contains(strings.ToLower(w), benignWarningAnchor) {
+			return false
+		}
+	}
+	return true
 }
 
 // IntentForScratchDestroy builds the benign teardown intent for an agent-owned scratch
@@ -152,16 +190,33 @@ func (e *Engine) runScratchTest(ctx context.Context, vmid int, spec RestoreTestS
 	}
 
 	// 3. Boot and verify it reaches running (basic liveness; deep app-health is slice 8).
+	//    The VERDICT is liveness (waitRunning), NEVER the start task's exitstatus. A start
+	//    that completes with warnings (e.g. the systemd-nesting advisory → exit "WARNINGS: N")
+	//    and then reaches running is a PASS — deciding pass/fail on an advisory exit code is
+	//    the crying-wolf bug this guards against. We pass AllowWarnings so WaitTask doesn't
+	//    hard-fail on it, then fetch + surface the warning text (visibility only).
 	startUPID, err := e.api.Start(ctx, vmid)
 	if err != nil {
 		res.Err = fmt.Errorf("reconcile: restore-test start: %w", err)
 		return
 	}
 	if startUPID != "" {
-		if _, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{}); err != nil {
+		st, err := e.api.WaitTask(ctx, startUPID, proxmox.WaitOptions{AllowWarnings: true})
+		if err != nil {
+			// A real (non-WARNINGS) start-task failure still fails the test.
 			res.Err = fmt.Errorf("reconcile: restore-test start task: %w", err)
 			return
 		}
+		if strings.HasPrefix(st.ExitStatus, "WARNINGS") {
+			// Surface the warning(s); do NOT fail. Liveness below is the verdict.
+			tail, logErr := e.api.TaskLogTail(ctx, startUPID, 50)
+			if logErr != nil {
+				e.logger.Warn("restore-test: could not read start-task log for warnings",
+					"vmid", vmid, "err", logErr)
+			}
+			res.StartWarnings = extractWarningLines(tail)
+			res.WarningsRecognized = warningsRecognized(res.StartWarnings)
+		}
 	}
 	if err := e.waitRunning(ctx, vmid, bootTimeout(spec)); err != nil {
 		res.Err = err

internal/reconcile/restoretest_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"strconv"
 	"testing"
 	"time"
 
@@ -55,6 +56,136 @@ func TestRunRestoreTest_PassAndTeardown(t *testing.T) {
 	}
 }
 
+// startWarnAPI builds a fakeAPI whose guest-start task exits "WARNINGS: 1" and whose start
+// task log contains the given warning lines. The guest reaches running (status default).
+func startWarnAPI(startUPID string, logLines []string) *fakeAPI {
+	return &fakeAPI{
+		cfg:       map[int]proxmox.GuestConfig{990000: scratchCfg()},
+		startUPID: startUPID,
+		waitFunc: func(upid string) (proxmox.TaskStatus, error) {
+			if upid == startUPID {
+				return proxmox.TaskStatus{Status: "stopped", ExitStatus: "WARNINGS: 1"}, nil
+			}
+			return proxmox.TaskStatus{Status: "stopped", ExitStatus: "OK"}, nil
+		},
+		logTailFunc: func(string) ([]string, error) { return logLines, nil },
+	}
+}
+
+func TestRunRestoreTest_PassWithRecognizedWarnings(t *testing.T) {
+	// The crux of the fix: start exits WARNINGS (systemd-nesting advisory) AND the guest
+	// reaches running → PASS. Warnings surfaced, recognized; verdict is liveness, not exit code.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{
+		"run_buffer: starting CT",
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"CT started",
+	})
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+	})
+	if !res.Pass || res.Err != nil {
+		t.Fatalf("start-with-warnings + running must PASS, got %+v", res)
+	}
+	if res.Verified != "boot+running" {
+		t.Errorf("verified = %q", res.Verified)
+	}
+	if len(res.StartWarnings) != 1 || !contains2(res.StartWarnings[0], "enable nesting") {
+		t.Fatalf("the nesting warning must be surfaced, got %+v", res.StartWarnings)
+	}
+	if !res.WarningsRecognized {
+		t.Errorf("the nesting warning must be recognized (benign)")
+	}
+}
+
+func TestRunRestoreTest_PassWithUnrecognizedWarning(t *testing.T) {
+	// An UNRECOGNIZED warning + running still PASSES (verdict is liveness), but is flagged
+	// not-recognized so the operator looks. Visibility-only, never a false-fail.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{"WARN: something unexpected during start"})
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+	})
+	if !res.Pass || res.Err != nil {
+		t.Fatalf("unrecognized warning + running must still PASS, got %+v", res)
+	}
+	if len(res.StartWarnings) != 1 {
+		t.Fatalf("warning must still be surfaced, got %+v", res.StartWarnings)
+	}
+	if res.WarningsRecognized {
+		t.Errorf("an unrecognized warning must NOT be recognized")
+	}
+}
+
+func TestRunRestoreTest_LivenessIsTheVerdict(t *testing.T) {
+	// Start exits WARNINGS but the guest NEVER reaches running → FAIL. The verdict is
+	// liveness; warnings can never turn a non-running guest into a pass.
+	const startUPID = "UPID:demo:start:990000:"
+	api := startWarnAPI(startUPID, []string{"WARN: Systemd 257 detected. You may need to enable nesting."})
+	api.status = map[int]proxmox.Guest{990000: {VMID: 990000, Status: "stopped"}}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunRestoreTest(context.Background(), RestoreTestSpec{
+		Archive: "vol", RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009,
+		BootTimeout: 40 * time.Millisecond,
+	})
+	if res.Pass || res.Err == nil {
+		t.Fatalf("not-running must FAIL regardless of warnings, got %+v", res)
+	}
+	if len(api.destroys) != 1 {
+		t.Errorf("teardown must still run: %+v", api.destroys)
+	}
+}
+
+// TestWarningsRecognized_VersionFree is the regression guard: the recognizer must match the
+// nesting advisory for the CURRENT systemd version AND future ones (258/259…), proving the
+// "enable nesting" anchor is version-independent and can't silently rot back into the bug.
+func TestWarningsRecognized_VersionFree(t *testing.T) {
+	for _, v := range []int{256, 257, 258, 259, 300} {
+		line := []string{"WARN: Systemd " + strconv.Itoa(v) + " detected. You may need to enable nesting."}
+		if !warningsRecognized(line) {
+			t.Errorf("systemd %d nesting advisory must be recognized (version-free anchor): %q", v, line[0])
+		}
+	}
+	// Empty ⇒ trivially recognized (N/A).
+	if !warningsRecognized(nil) {
+		t.Error("empty warnings must be trivially recognized")
+	}
+	// An unrelated warning is NOT recognized.
+	if warningsRecognized([]string{"WARN: disk nearly full"}) {
+		t.Error("an unrelated warning must not be recognized")
+	}
+	// Mixed: one benign + one unrelated ⇒ NOT recognized (every line must match).
+	if warningsRecognized([]string{
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"WARN: disk nearly full",
+	}) {
+		t.Error("a mix with any unrecognized line must not be recognized")
+	}
+}
+
+func TestExtractWarningLines(t *testing.T) {
+	got := extractWarningLines([]string{
+		"run_buffer: starting",
+		"WARN: Systemd 257 detected. You may need to enable nesting.",
+		"  WARN: indented warning  ",
+		"INFO: not a warning",
+	})
+	if len(got) != 2 {
+		t.Fatalf("want 2 warning lines, got %d: %+v", len(got), got)
+	}
+	if !contains2(got[0], "enable nesting") || got[1] != "WARN: indented warning" {
+		t.Errorf("warning extraction/trim wrong: %+v", got)
+	}
+}
+
 func TestRunRestoreTest_TeardownOnFailedVerify(t *testing.T) {
 	// Guest never reaches running → verify fails, but teardown MUST still run.
 	api := &fakeAPI{

internal/reconcile/state.go

@@ -126,6 +126,10 @@ type GuestAPI interface {
 	// TaskStatusOnce is a single non-blocking task-status read — used by crash
 	// recovery to learn the outcome of an op that was in flight when the agent died.
 	TaskStatusOnce(ctx context.Context, upid string) (proxmox.TaskStatus, error)
+	// TaskLogTail fetches up to limit trailing task-log lines — used to surface a guest-start
+	// task's warning text (the restore-test fetches it when the start exits "WARNINGS: N", so
+	// the advisory is reported without failing a guest that actually boots).
+	TaskLogTail(ctx context.Context, upid string, limit int) ([]string, error)
 }
 
 // guestDescription decodes the (string-valued) `description` key from a GuestConfig's