hub v0.179.1: a host boot ends "the host needs a restart" at once (demo-felhom 2026-11-08)
gates / gates (push) Successful in 2m3s
gates / gates (push) Successful in 2m3s
The host layer's reboot-needed run is cut at the host's last boot (crash guard last_boot_at, one minute skew margin): the kernel step's reboot is reported on the kernel layer, so the flag outlived it until the next night. Feeds the System page and the 14-day alarm. Test TestRebootNeeded_ClearedByAHostBootAfterTheClaim, red on two mutants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
a0d8057634
commit
ddbcabebb4
3 files changed
+135
-6
No files matched your search
@@ -1,3 +1,18 @@
|
|||||||
|
## v0.179.1 (2026-11-08) — a host reboot ends „the host needs a restart" at once
|
||||||
|
|
||||||
|
- **The defect (demo-felhom, 2026-11-08).** The night's host pass said reboot-needed (05:31 CET); the kernel step rebooted
|
||||||
|
the box at 05:35; the System page still read „the host needs a restart since 2026-11-08" in the morning. The reboot is
|
||||||
|
reported on the KERNEL layer (or not at all, by hand) and the host layer's need lasted until the next night's host pass.
|
||||||
|
Checked on the box: no host process maps a replaced library after the boot.
|
||||||
|
- **The fix (`osupdates.layerLine`).** For the host layer, the host's last boot (the crash guard's `last_boot_at` in the
|
||||||
|
newest host report) cuts the reboot-needed run: every claim the hub received before the boot, less one minute for clock
|
||||||
|
skew, is ended; a claim after the boot starts a new run. No boot fact = the old behaviour. The same line feeds the System
|
||||||
|
page and the 14-day `reboot_needed` alarm.
|
||||||
|
- **Test** `TestRebootNeeded_ClearedByAHostBootAfterTheClaim` (boot before the claim, inside the margin, after it, then a
|
||||||
|
new claim); red seen on two mutants (no cut; the margin's sign flipped). `go test ./...` green.
|
||||||
|
- **Live facts it will act on:** demo-felhom boot 04:35:18Z after its claim at ~04:31Z → cleared; demo-hp boot
|
||||||
|
2026-10-29 before its 2026-11-08 claim → stays (it does need a restart).
|
||||||
|
|
||||||
## v0.179.0 (2026-11-03) — no „does not answer" alarm after the restart mail of the same episode (R-1352)
|
## v0.179.0 (2026-11-03) — no „does not answer" alarm after the restart mail of the same episode (R-1352)
|
||||||
|
|
||||||
- **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2
|
- **R-1352 — the „does not answer" alarm is not mailed after the restart mail of the same episode.** The alarm and the restart lie 2
|
||||||
|
|||||||
@@ -328,3 +328,60 @@ func TestRebootNeeded_ClearedByAScannedPass(t *testing.T) {
|
|||||||
t.Fatal("reboot alarm after the reboot")
|
t.Fatal("reboot alarm after the reboot")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bootReport stores a host report whose crash guard says the host last booted at `boot` (the agent's
|
||||||
|
// system.facts.host.crash_guard.last_boot_at, UTC ISO).
|
||||||
|
func bootReport(t *testing.T, f *fix, host string, boot time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
h, err := f.s.Store.GetHost(host)
|
||||||
|
if err != nil || h == nil {
|
||||||
|
t.Fatalf("no host %s", host)
|
||||||
|
}
|
||||||
|
body := `{"system":{"pve_version":"pve-manager/9.2.2/x","facts":{"host":{"crash_guard":{"last_boot_at":"` +
|
||||||
|
boot.UTC().Format("2006-01-02T15:04:05Z") + `"}}}}}`
|
||||||
|
if err := f.s.Store.SaveHostReport(host, h.CustomerID, []byte(body), store.HostReportDenorm{AgentVersion: "0.172.1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2026-11-08, demo-felhom: the night's host pass said reboot-needed at 05:31, the kernel step rebooted the box at
|
||||||
|
// 05:35, and the System page kept „the host needs a restart" until the next night's host pass — the reboot is
|
||||||
|
// reported on the KERNEL layer, never the host layer. A host boot AFTER the newest claim ends the need, on the page
|
||||||
|
// AND for the 14-day alarm. A boot BEFORE the claim (or within the margin) does not. Red-proof: drop the boot check
|
||||||
|
// in layerLine and the first assertion fails.
|
||||||
|
func TestRebootNeeded_ClearedByAHostBootAfterTheClaim(t *testing.T) {
|
||||||
|
f := newFix(t)
|
||||||
|
if _, err := f.s.Store.GetHost("hp"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true,
|
||||||
|
Upgraded: []Package{pk("libc6", "2")}, RebootNeeded: true, RebootScanned: true})
|
||||||
|
claim := f.now
|
||||||
|
// A boot BEFORE the claim says nothing: the need stays.
|
||||||
|
bootReport(t, f, "hp", claim.Add(-time.Hour))
|
||||||
|
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() {
|
||||||
|
t.Fatal("a boot before the claim cleared the need")
|
||||||
|
}
|
||||||
|
// A boot within the margin after the claim (clock skew) says nothing either.
|
||||||
|
bootReport(t, f, "hp", claim.Add(30*time.Second))
|
||||||
|
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() {
|
||||||
|
t.Fatal("a boot inside the skew margin cleared the need")
|
||||||
|
}
|
||||||
|
// The kernel step's reboot, 4 minutes after the claim: the need is over.
|
||||||
|
f.now = f.now.Add(10 * time.Minute)
|
||||||
|
bootReport(t, f, "hp", claim.Add(4*time.Minute))
|
||||||
|
if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.IsZero() {
|
||||||
|
t.Fatalf("a host boot after the claim did not clear the need: since %v", l.RebootNeededSince)
|
||||||
|
}
|
||||||
|
f.now = f.now.Add(20 * 24 * time.Hour)
|
||||||
|
f.s.Alarms()
|
||||||
|
if f.count(EventRebootNeeded) != 0 {
|
||||||
|
t.Fatal("reboot alarm after the host rebooted")
|
||||||
|
}
|
||||||
|
// A NEW claim after that boot (the next night installs more) starts the need again.
|
||||||
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true,
|
||||||
|
Upgraded: []Package{pk("libc6", "3")}, RebootNeeded: true, RebootScanned: true})
|
||||||
|
if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.Equal(f.now) {
|
||||||
|
t.Fatalf("a claim after the boot: since = %v, want %v", l.RebootNeededSince, f.now)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -938,15 +938,59 @@ func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
|
|||||||
l.LastSuccessfulLeg = ok.ReceivedAt
|
l.LastSuccessfulLeg = ok.ReceivedAt
|
||||||
}
|
}
|
||||||
l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
|
l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
|
||||||
l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) {
|
pred := func(x Report) (bool, bool) {
|
||||||
if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" {
|
if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" {
|
||||||
return false, false // a pass that did not look says nothing about reboot-needed: skip it
|
return false, false // a pass that did not look says nothing about reboot-needed: skip it
|
||||||
}
|
}
|
||||||
return x.RebootNeeded, true
|
return x.RebootNeeded, true
|
||||||
})
|
}
|
||||||
|
// 2026-11-08 (demo-felhom): a host reboot is reported on the KERNEL layer (or not at all, by hand), never on the host
|
||||||
|
// layer, so the host layer's need outlived the reboot that ended it until the next night's host pass. A host boot
|
||||||
|
// ends every claim the hub received before it: the run is cut at the boot — for the System page and the 14-day
|
||||||
|
// alarm alike; a claim after the boot starts a new run. The boot is a positive fact (the crash guard's
|
||||||
|
// last_boot_at, box clock) compared with the hub's receipt, less a margin for clock skew (a claim inside the margin
|
||||||
|
// still counts); no boot fact = the old behaviour. Pinned by TestRebootNeeded_ClearedByAHostBootAfterTheClaim.
|
||||||
|
var cut time.Time
|
||||||
|
if layer == LayerHost {
|
||||||
|
if boot := s.hostLastBoot(hostID); !boot.IsZero() {
|
||||||
|
cut = boot.Add(-rebootClaimSkew)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var whole time.Time
|
||||||
|
l.RebootNeededSince, whole = s.runSinceAfter(hostID, layer, pred, cut)
|
||||||
|
if l.RebootNeededSince.IsZero() && !whole.IsZero() {
|
||||||
|
s.logf("[DEBUG] osupdates: %s reboot-needed (since %s) ended by the host boot (cut %s)",
|
||||||
|
hostID, whole.UTC().Format(time.RFC3339), cut.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
return l
|
return l
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rebootClaimSkew: a host boot must come this long after the hub received the newest reboot-needed claim to end it
|
||||||
|
// (the boot time is the box's clock, the receipt the hub's; R-1301 measured a box 14 s slow).
|
||||||
|
const rebootClaimSkew = time.Minute
|
||||||
|
|
||||||
|
// hostLastBoot is the host's last boot as its crash guard reports it (zero: no host report, no crash guard, or an
|
||||||
|
// unreadable time — the caller then keeps the need).
|
||||||
|
func (s *Service) hostLastBoot(hostID string) time.Time {
|
||||||
|
h, err := s.Store.GetHost(hostID)
|
||||||
|
if err != nil || h == nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID)
|
||||||
|
if rj == "" {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
cg := sysfacts.Parse(rj).Host.CrashGuard
|
||||||
|
if cg == nil || cg.LastBootAt == "" {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
t, err := time.Parse(time.RFC3339, cg.LastBootAt)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
|
// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
|
||||||
// installs every fast-lane update itself, so it never has any.
|
// installs every fast-lane update itself, so it never has any.
|
||||||
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
|
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
|
||||||
@@ -972,11 +1016,18 @@ func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
|
|||||||
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
|
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
|
||||||
// false = the report does not count either way (skipped).
|
// false = the report does not count either way (skipped).
|
||||||
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
|
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
|
||||||
|
since, _ := s.runSinceAfter(hostID, layer, pred, time.Time{})
|
||||||
|
return since
|
||||||
|
}
|
||||||
|
|
||||||
|
// runSinceAfter is sinceContinuously with a cut: a report received at or before `cut` ends the run (zero cut = no cut).
|
||||||
|
// It also returns the uncut run's start, so a caller can tell „ended by the cut" from „never held".
|
||||||
|
func (s *Service) runSinceAfter(hostID, layer string, pred func(Report) (bool, bool), cut time.Time) (since, uncut time.Time) {
|
||||||
reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
|
reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return time.Time{}
|
return time.Time{}, time.Time{}
|
||||||
}
|
}
|
||||||
var since time.Time
|
cutHit := false
|
||||||
for _, rep := range reps {
|
for _, rep := range reps {
|
||||||
var r Report
|
var r Report
|
||||||
if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
|
if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
|
||||||
@@ -989,9 +1040,15 @@ func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (boo
|
|||||||
if !hold {
|
if !hold {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
since = rep.ReceivedAt
|
uncut = rep.ReceivedAt
|
||||||
|
if !cutHit && !cut.IsZero() && !rep.ReceivedAt.After(cut) {
|
||||||
|
cutHit = true
|
||||||
|
}
|
||||||
|
if !cutHit {
|
||||||
|
since = rep.ReceivedAt
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return since
|
return since, uncut
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fleet lists every box.
|
// Fleet lists every box.
|
||||||
|
|||||||
Reference in new issue
Block a user