hub v0.179.1: a host boot ends "the host needs a restart" at once (demo-felhom 2026-11-08)
gates / gates (push) Successful in 2m3s

The host layer's reboot-needed run is cut at the host's last boot (crash guard
last_boot_at, one minute skew margin): the kernel step's reboot is reported on the
kernel layer, so the flag outlived it until the next night. Feeds the System page
and the 14-day alarm. Test TestRebootNeeded_ClearedByAHostBootAfterTheClaim, red on
two mutants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-11-08 07:21:50 +01:00
1 parent a0d8057634
commit ddbcabebb4
3 files changed
+135 -6

No files matched your search

+57
View File
@@ -328,3 +328,60 @@ func TestRebootNeeded_ClearedByAScannedPass(t *testing.T) {
t.Fatal("reboot alarm after the reboot")
}
}
// bootReport stores a host report whose crash guard says the host last booted at `boot` (the agent's
// system.facts.host.crash_guard.last_boot_at, UTC ISO).
func bootReport(t *testing.T, f *fix, host string, boot time.Time) {
t.Helper()
h, err := f.s.Store.GetHost(host)
if err != nil || h == nil {
t.Fatalf("no host %s", host)
}
body := `{"system":{"pve_version":"pve-manager/9.2.2/x","facts":{"host":{"crash_guard":{"last_boot_at":"` +
boot.UTC().Format("2006-01-02T15:04:05Z") + `"}}}}}`
if err := f.s.Store.SaveHostReport(host, h.CustomerID, []byte(body), store.HostReportDenorm{AgentVersion: "0.172.1"}); err != nil {
t.Fatal(err)
}
}
// 2026-11-08, demo-felhom: the night's host pass said reboot-needed at 05:31, the kernel step rebooted the box at
// 05:35, and the System page kept „the host needs a restart" until the next night's host pass — the reboot is
// reported on the KERNEL layer, never the host layer. A host boot AFTER the newest claim ends the need, on the page
// AND for the 14-day alarm. A boot BEFORE the claim (or within the margin) does not. Red-proof: drop the boot check
// in layerLine and the first assertion fails.
func TestRebootNeeded_ClearedByAHostBootAfterTheClaim(t *testing.T) {
f := newFix(t)
if _, err := f.s.Store.GetHost("hp"); err != nil {
t.Fatal(err)
}
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true,
Upgraded: []Package{pk("libc6", "2")}, RebootNeeded: true, RebootScanned: true})
claim := f.now
// A boot BEFORE the claim says nothing: the need stays.
bootReport(t, f, "hp", claim.Add(-time.Hour))
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() {
t.Fatal("a boot before the claim cleared the need")
}
// A boot within the margin after the claim (clock skew) says nothing either.
bootReport(t, f, "hp", claim.Add(30*time.Second))
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() {
t.Fatal("a boot inside the skew margin cleared the need")
}
// The kernel step's reboot, 4 minutes after the claim: the need is over.
f.now = f.now.Add(10 * time.Minute)
bootReport(t, f, "hp", claim.Add(4*time.Minute))
if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.IsZero() {
t.Fatalf("a host boot after the claim did not clear the need: since %v", l.RebootNeededSince)
}
f.now = f.now.Add(20 * 24 * time.Hour)
f.s.Alarms()
if f.count(EventRebootNeeded) != 0 {
t.Fatal("reboot alarm after the host rebooted")
}
// A NEW claim after that boot (the next night installs more) starts the need again.
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true,
Upgraded: []Package{pk("libc6", "3")}, RebootNeeded: true, RebootScanned: true})
if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.Equal(f.now) {
t.Fatalf("a claim after the boot: since = %v, want %v", l.RebootNeededSince, f.now)
}
}
+63 -6
View File
@@ -938,15 +938,59 @@ func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
l.LastSuccessfulLeg = ok.ReceivedAt
}
l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) {
pred := func(x Report) (bool, bool) {
if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" {
return false, false // a pass that did not look says nothing about reboot-needed: skip it
}
return x.RebootNeeded, true
})
}
// 2026-11-08 (demo-felhom): a host reboot is reported on the KERNEL layer (or not at all, by hand), never on the host
// layer, so the host layer's need outlived the reboot that ended it until the next night's host pass. A host boot
// ends every claim the hub received before it: the run is cut at the boot — for the System page and the 14-day
// alarm alike; a claim after the boot starts a new run. The boot is a positive fact (the crash guard's
// last_boot_at, box clock) compared with the hub's receipt, less a margin for clock skew (a claim inside the margin
// still counts); no boot fact = the old behaviour. Pinned by TestRebootNeeded_ClearedByAHostBootAfterTheClaim.
var cut time.Time
if layer == LayerHost {
if boot := s.hostLastBoot(hostID); !boot.IsZero() {
cut = boot.Add(-rebootClaimSkew)
}
}
var whole time.Time
l.RebootNeededSince, whole = s.runSinceAfter(hostID, layer, pred, cut)
if l.RebootNeededSince.IsZero() && !whole.IsZero() {
s.logf("[DEBUG] osupdates: %s reboot-needed (since %s) ended by the host boot (cut %s)",
hostID, whole.UTC().Format(time.RFC3339), cut.UTC().Format(time.RFC3339))
}
return l
}
// rebootClaimSkew: a host boot must come this long after the hub received the newest reboot-needed claim to end it
// (the boot time is the box's clock, the receipt the hub's; R-1301 measured a box 14 s slow).
const rebootClaimSkew = time.Minute
// hostLastBoot is the host's last boot as its crash guard reports it (zero: no host report, no crash guard, or an
// unreadable time — the caller then keeps the need).
func (s *Service) hostLastBoot(hostID string) time.Time {
h, err := s.Store.GetHost(hostID)
if err != nil || h == nil {
return time.Time{}
}
rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID)
if rj == "" {
return time.Time{}
}
cg := sysfacts.Parse(rj).Host.CrashGuard
if cg == nil || cg.LastBootAt == "" {
return time.Time{}
}
t, err := time.Parse(time.RFC3339, cg.LastBootAt)
if err != nil {
return time.Time{}
}
return t
}
// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
// installs every fast-lane update itself, so it never has any.
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
@@ -972,11 +1016,18 @@ func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
// false = the report does not count either way (skipped).
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
since, _ := s.runSinceAfter(hostID, layer, pred, time.Time{})
return since
}
// runSinceAfter is sinceContinuously with a cut: a report received at or before `cut` ends the run (zero cut = no cut).
// It also returns the uncut run's start, so a caller can tell „ended by the cut" from „never held".
func (s *Service) runSinceAfter(hostID, layer string, pred func(Report) (bool, bool), cut time.Time) (since, uncut time.Time) {
reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
if err != nil {
return time.Time{}
return time.Time{}, time.Time{}
}
var since time.Time
cutHit := false
for _, rep := range reps {
var r Report
if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
@@ -989,9 +1040,15 @@ func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (boo
if !hold {
break
}
since = rep.ReceivedAt
uncut = rep.ReceivedAt
if !cutHit && !cut.IsZero() && !rep.ReceivedAt.After(cut) {
cutHit = true
}
if !cutHit {
since = rep.ReceivedAt
}
}
return since
return since, uncut
}
// Fleet lists every box.