restore-test: verdict is liveness, not start-task exitstatus (v0.7.0)
Fixes the crying-wolf false-fail surfaced by the live hub-enrollment runbook: PVE's guest-start task exits "WARNINGS: 1" for the benign systemd-nesting advisory, and WaitTask treated any non-OK exitstatus as failure, so the verdict was decided by an advisory exit code before the real boot check ran. Every modern-distro restore-test reported pass:false. - proxmox.WaitOptions.AllowWarnings (opt-in; default keeps all callers strict) - restore-test start step accepts warnings, surfaces them, verdict stays waitRunning - RestoreTestResult.StartWarnings/.WarningsRecognized + version-free "enable nesting" recognizer (can't rot back at systemd 258+); GuestAPI.TaskLogTail - hub.RestoreTest.warnings/.warnings_recognized wire fields (consumed by hub v0.7.5) - scheduler logs clean / passed-with-recognized / passed-with-unrecognized warnings - tests: WaitTask warnings matrix; restore-test pass/fail-on-liveness; version-free regression guard (systemd 256-300) Single agent bump 0.6.0 -> 0.7.0 covering the agent half of both task phases. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -205,13 +205,15 @@ func mountpointLabel(key, cfg string) string {
|
||||
// package owns the reconcile→hub mapping so reconcile need not import hub for the result).
|
||||
func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.RestoreTest {
|
||||
rt := hub.RestoreTest{
|
||||
SourceArchive: res.Archive,
|
||||
SourceTier: res.SourceTier,
|
||||
ScratchVMID: res.ScratchVMID,
|
||||
Pass: res.Pass,
|
||||
Verified: res.Verified,
|
||||
TestedAt: testedAt.Format(time.RFC3339),
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
SourceArchive: res.Archive,
|
||||
SourceTier: res.SourceTier,
|
||||
ScratchVMID: res.ScratchVMID,
|
||||
Pass: res.Pass,
|
||||
Verified: res.Verified,
|
||||
TestedAt: testedAt.Format(time.RFC3339),
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
Warnings: res.StartWarnings,
|
||||
WarningsRecognized: res.WarningsRecognized,
|
||||
}
|
||||
if res.Err != nil {
|
||||
rt.Error = res.Err.Error()
|
||||
|
||||
@@ -102,10 +102,19 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
}
|
||||
rt := ToHubRestoreTest(res, s.now())
|
||||
s.store.RecordRestoreTest(rt)
|
||||
if rt.Pass {
|
||||
s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
|
||||
} else {
|
||||
switch {
|
||||
case !rt.Pass:
|
||||
// A failing restore-test is the loudest DR signal there is.
|
||||
s.logger.Error("backup: scheduled restore-test FAILED", "archive", rt.SourceArchive, "err", rt.Error)
|
||||
case len(res.StartWarnings) == 0:
|
||||
s.logger.Info("backup: scheduled restore-test passed", "archive", rt.SourceArchive, "duration_s", rt.DurationSeconds)
|
||||
case res.WarningsRecognized:
|
||||
// Passed; the only warnings are the known-benign (e.g. systemd-nesting) advisory.
|
||||
s.logger.Info("backup: scheduled restore-test passed with warnings (recognized)",
|
||||
"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
|
||||
default:
|
||||
// Passed liveness, but an UNRECOGNIZED start warning stood out — worth an operator look.
|
||||
s.logger.Warn("backup: scheduled restore-test passed with UNRECOGNIZED warnings",
|
||||
"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user