From ddbcabebb4e380e4c8c679efd92a3aa10d5f2dbe Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 8 Nov 2026 07:21:50 +0100 Subject: [PATCH] hub v0.179.1: a host boot ends "the host needs a restart" at once (demo-felhom 2026-11-08) The host layer's reboot-needed run is cut at the host's last boot (crash guard last_boot_at, one minute skew margin): the kernel step's reboot is reported on the kernel layer, so the flag outlived it until the next night. Feeds the System page and the 14-day alarm. Test TestRebootNeeded_ClearedByAHostBootAfterTheClaim, red on two mutants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- hub/CHANGELOG.md | 15 ++++++ hub/internal/osupdates/alarms_test.go | 57 ++++++++++++++++++++++ hub/internal/osupdates/service.go | 69 ++++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index a4b2556e..2d97a28a 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,18 @@ +## v0.179.1 (2026-11-08) — a host reboot ends „the host needs a restart" at once + +- **The defect (demo-felhom, 2026-11-08).** The night's host pass said reboot-needed (05:31 CET); the kernel step rebooted + the box at 05:35; the System page still read „the host needs a restart since 2026-11-08" in the morning. The reboot is + reported on the KERNEL layer (or not at all, by hand) and the host layer's need lasted until the next night's host pass. + Checked on the box: no host process maps a replaced library after the boot. +- **The fix (`osupdates.layerLine`).** For the host layer, the host's last boot (the crash guard's `last_boot_at` in the + newest host report) cuts the reboot-needed run: every claim the hub received before the boot, less one minute for clock + skew, is ended; a claim after the boot starts a new run. No boot fact = the old behaviour. The same line feeds the System + page and the 14-day `reboot_needed` alarm. +- **Test** `TestRebootNeeded_ClearedByAHostBootAfterTheClaim` (boot before the claim, inside the margin, after it, then a + new claim); red seen on two mutants (no cut; the margin's sign flipped). `go test ./...` green. +- **Live facts it will act on:** demo-felhom boot 04:35:18Z after its claim at ~04:31Z → cleared; demo-hp boot + 2026-10-29 before its 2026-11-08 claim → stays (it does need a restart). + ## v0.179.0 (2026-11-03) — no „does not answer" alarm after the restart mail of the same episode (R-1352) - **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2 diff --git a/hub/internal/osupdates/alarms_test.go b/hub/internal/osupdates/alarms_test.go index 30c994ec..b4b7e49d 100644 --- a/hub/internal/osupdates/alarms_test.go +++ b/hub/internal/osupdates/alarms_test.go @@ -328,3 +328,60 @@ func TestRebootNeeded_ClearedByAScannedPass(t *testing.T) { t.Fatal("reboot alarm after the reboot") } } + +// bootReport stores a host report whose crash guard says the host last booted at `boot` (the agent's +// system.facts.host.crash_guard.last_boot_at, UTC ISO). +func bootReport(t *testing.T, f *fix, host string, boot time.Time) { + t.Helper() + h, err := f.s.Store.GetHost(host) + if err != nil || h == nil { + t.Fatalf("no host %s", host) + } + body := `{"system":{"pve_version":"pve-manager/9.2.2/x","facts":{"host":{"crash_guard":{"last_boot_at":"` + + boot.UTC().Format("2006-01-02T15:04:05Z") + `"}}}}}` + if err := f.s.Store.SaveHostReport(host, h.CustomerID, []byte(body), store.HostReportDenorm{AgentVersion: "0.172.1"}); err != nil { + t.Fatal(err) + } +} + +// 2026-11-08, demo-felhom: the night's host pass said reboot-needed at 05:31, the kernel step rebooted the box at +// 05:35, and the System page kept „the host needs a restart" until the next night's host pass — the reboot is +// reported on the KERNEL layer, never the host layer. A host boot AFTER the newest claim ends the need, on the page +// AND for the 14-day alarm. A boot BEFORE the claim (or within the margin) does not. Red-proof: drop the boot check +// in layerLine and the first assertion fails. +func TestRebootNeeded_ClearedByAHostBootAfterTheClaim(t *testing.T) { + f := newFix(t) + if _, err := f.s.Store.GetHost("hp"); err != nil { + t.Fatal(err) + } + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, + Upgraded: []Package{pk("libc6", "2")}, RebootNeeded: true, RebootScanned: true}) + claim := f.now + // A boot BEFORE the claim says nothing: the need stays. + bootReport(t, f, "hp", claim.Add(-time.Hour)) + if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() { + t.Fatal("a boot before the claim cleared the need") + } + // A boot within the margin after the claim (clock skew) says nothing either. + bootReport(t, f, "hp", claim.Add(30*time.Second)) + if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() { + t.Fatal("a boot inside the skew margin cleared the need") + } + // The kernel step's reboot, 4 minutes after the claim: the need is over. + f.now = f.now.Add(10 * time.Minute) + bootReport(t, f, "hp", claim.Add(4*time.Minute)) + if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.IsZero() { + t.Fatalf("a host boot after the claim did not clear the need: since %v", l.RebootNeededSince) + } + f.now = f.now.Add(20 * 24 * time.Hour) + f.s.Alarms() + if f.count(EventRebootNeeded) != 0 { + t.Fatal("reboot alarm after the host rebooted") + } + // A NEW claim after that boot (the next night installs more) starts the need again. + f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, + Upgraded: []Package{pk("libc6", "3")}, RebootNeeded: true, RebootScanned: true}) + if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.Equal(f.now) { + t.Fatalf("a claim after the boot: since = %v, want %v", l.RebootNeededSince, f.now) + } +} diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go index cee80549..4cbdb165 100644 --- a/hub/internal/osupdates/service.go +++ b/hub/internal/osupdates/service.go @@ -938,15 +938,59 @@ func (s *Service) layerLine(hostID, layer string, ring int) LayerLine { l.LastSuccessfulLeg = ok.ReceivedAt } l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r)) - l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) { + pred := func(x Report) (bool, bool) { if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" { return false, false // a pass that did not look says nothing about reboot-needed: skip it } return x.RebootNeeded, true - }) + } + // 2026-11-08 (demo-felhom): a host reboot is reported on the KERNEL layer (or not at all, by hand), never on the host + // layer, so the host layer's need outlived the reboot that ended it until the next night's host pass. A host boot + // ends every claim the hub received before it: the run is cut at the boot — for the System page and the 14-day + // alarm alike; a claim after the boot starts a new run. The boot is a positive fact (the crash guard's + // last_boot_at, box clock) compared with the hub's receipt, less a margin for clock skew (a claim inside the margin + // still counts); no boot fact = the old behaviour. Pinned by TestRebootNeeded_ClearedByAHostBootAfterTheClaim. + var cut time.Time + if layer == LayerHost { + if boot := s.hostLastBoot(hostID); !boot.IsZero() { + cut = boot.Add(-rebootClaimSkew) + } + } + var whole time.Time + l.RebootNeededSince, whole = s.runSinceAfter(hostID, layer, pred, cut) + if l.RebootNeededSince.IsZero() && !whole.IsZero() { + s.logf("[DEBUG] osupdates: %s reboot-needed (since %s) ended by the host boot (cut %s)", + hostID, whole.UTC().Format(time.RFC3339), cut.UTC().Format(time.RFC3339)) + } return l } +// rebootClaimSkew: a host boot must come this long after the hub received the newest reboot-needed claim to end it +// (the boot time is the box's clock, the receipt the hub's; R-1301 measured a box 14 s slow). +const rebootClaimSkew = time.Minute + +// hostLastBoot is the host's last boot as its crash guard reports it (zero: no host report, no crash guard, or an +// unreadable time — the caller then keeps the need). +func (s *Service) hostLastBoot(hostID string) time.Time { + h, err := s.Store.GetHost(hostID) + if err != nil || h == nil { + return time.Time{} + } + rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID) + if rj == "" { + return time.Time{} + } + cg := sysfacts.Parse(rj).Host.CrashGuard + if cg == nil || cg.LastBootAt == "" { + return time.Time{} + } + t, err := time.Parse(time.RFC3339, cg.LastBootAt) + if err != nil { + return time.Time{} + } + return t +} + // notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0 // installs every fast-lane update itself, so it never has any. func (s *Service) notCoveredFast(layer string, ring int, r Report) []string { @@ -972,11 +1016,18 @@ func (s *Service) notCoveredFast(layer string, ring int, r Report) []string { // current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result // false = the report does not count either way (skipped). func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time { + since, _ := s.runSinceAfter(hostID, layer, pred, time.Time{}) + return since +} + +// runSinceAfter is sinceContinuously with a cut: a report received at or before `cut` ends the run (zero cut = no cut). +// It also returns the uncut run's start, so a caller can tell „ended by the cut" from „never held". +func (s *Service) runSinceAfter(hostID, layer string, pred func(Report) (bool, bool), cut time.Time) (since, uncut time.Time) { reps, err := s.Store.OSReportsDesc(hostID, layer, 400) if err != nil { - return time.Time{} + return time.Time{}, time.Time{} } - var since time.Time + cutHit := false for _, rep := range reps { var r Report if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil { @@ -989,9 +1040,15 @@ func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (boo if !hold { break } - since = rep.ReceivedAt + uncut = rep.ReceivedAt + if !cutHit && !cut.IsZero() && !rep.ReceivedAt.After(cut) { + cutHit = true + } + if !cutHit { + since = rep.ReceivedAt + } } - return since + return since, uncut } // Fleet lists every box.