hub (unreleased): the operator is told when a box's clock is more than one minute from the hub's (R-1384, D79)
gates / gates (push) Successful in 2m11s

The host report's new sent_at (the box clock at the moment of sending) is compared with the hub's clock at arrival.
Both directions; told once; clears within 30 s. A report that waited on the box raises nothing (reported_at is not
read). A second box off in the same direction points at the hub's own clock. Operator-only. Not deployed.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
admin committed 2026-11-14 17:05:40 +01:00
1 parent 26f4f7660c
commit 88e636445a
9 files changed
+318 -2

No files matched your search

+92 -1
View File
@@ -778,7 +778,11 @@ type hostReportPayload struct {
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
Guests []struct {
// SentAt (R-1384, D79): the box's clock at the moment the agent SENDS the report (stamped in the agent's
// Client.Report, after the collect). "" = an agent that does not stamp it → the clock is not judged.
// reported_at is NOT used for this: it is stamped when the collect starts, and a collect can take long.
SentAt string `json:"sent_at"`
Guests []struct {
VMID int `json:"vmid"`
Name string `json:"name"`
Status string `json:"status"`
@@ -942,6 +946,7 @@ type hostStorageTarget struct {
// handleHostReport ingests the agent's host-report (the heartbeat) and returns the
// control envelope (agent spec §5).
func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
received := time.Now() // before the body is read: the clock check (R-1384) compares the box's send time with THIS
hostID, custID, isGlobal, ok := h.checkAuthHost(r)
if !ok {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
@@ -1012,6 +1017,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
h.checkWGTunnel(hostID, custID, rep.Wireguard)
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
h.checkBoxClock(hostID, custID, rep.SentAt, received)
h.checkCrash(hostID, custID, body)
h.checkDeliberateOff(hostID, custID, &rep)
@@ -3240,6 +3246,91 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
}
}
// Box clock alarm (R-1384, D79, `09` §3 decision 300; `08-alarm-ladder.md`). The hub compares the box's clock at
// the moment of SENDING (`sent_at`) with its own clock at the moment the request arrives. More than one minute apart,
// in either direction → one operator-only `box_clock_wrong`; back within 30 seconds → one `box_clock_ok` (the gap
// between 30 and 60 s keeps a clock at the edge from mailing at every report).
//
// What this can NOT tell: whose clock is wrong. A slow line can only make a box look BEHIND, and by less than the
// agent's request timeout (30 s), so it cannot raise the alarm alone. When the HUB's clock is wrong, every healthy box
// looks wrong in the SAME direction — so the message says so when a second box is off in the same direction.
// A report with no `sent_at` (an older agent) is never judged. Pinned by TestBoxClock_*.
const (
eventBoxClockWrong = "box_clock_wrong"
eventBoxClockOK = "box_clock_ok"
boxClockRaiseAbove = time.Minute
boxClockClearBelow = 30 * time.Second
)
func (h *Handler) checkBoxClock(hostID, custID, sentAt string, received time.Time) {
if sentAt == "" {
return
}
sent, err := time.Parse(time.RFC3339Nano, sentAt)
if err != nil {
h.logger.Printf("[WARN] host %s: sent_at is not a time (%d bytes); the clock is not judged", hostID, len(sentAt))
return
}
skew := sent.Sub(received) // > 0: the box is AHEAD of the hub
abs := skew
if abs < 0 {
abs = -abs
}
dir := "ahead"
if skew < 0 {
dir = "behind"
}
key := func(d string) string { return "box_clock:" + hostID + ":" + d }
emit := func(typ, sev, msg string, details map[string]any) {
dj, _ := json.Marshal(details)
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
}
}
raised := ""
for _, d := range []string{"ahead", "behind"} {
if !h.store.OSAlarmRaised(key(d)).IsZero() {
raised = d
}
}
secs := int64(abs / time.Second)
switch {
case abs > boxClockRaiseAbove:
if raised == dir {
return // said once
}
if raised != "" {
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
}
_ = h.store.SetOSAlarmRaised(key(dir), received)
// the hub's own clock: another box off in the same direction points at the hub, not at the boxes
var same []string
if hosts, lerr := h.store.ListHosts(); lerr == nil {
for _, o := range hosts {
if o.HostID != hostID && !h.store.OSAlarmRaised("box_clock:"+o.HostID+":"+dir).IsZero() {
same = append(same, o.HostID)
}
}
}
sort.Strings(same)
hint := ""
if len(same) > 0 {
hint = fmt.Sprintf(" %d other box(es) are %s too (%s): check the HUB's own clock (DooPlex) first.", len(same), dir, strings.Join(same, ", "))
}
h.logger.Printf("[WARN] host %s clock: %d s %s of the hub (same direction: %d other)", hostID, secs, dir, len(same))
emit(eventBoxClockWrong, "warning",
fmt.Sprintf("The clock of %s is %d seconds %s the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time. Read its time service (`chronyc tracking`).%s", hostID, secs, map[string]string{"ahead": "ahead of", "behind": "behind"}[dir], hint),
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "direction": dir, "same_direction": same})
case abs <= boxClockClearBelow && raised != "":
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
h.logger.Printf("[INFO] host %s clock: in step again (%d s; was %s)", hostID, secs, raised)
emit(eventBoxClockOK, "info", fmt.Sprintf("The clock of %s is in step with the hub's clock again (was %s).", hostID, raised),
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "was": raised})
}
}
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,