hub (unreleased): the operator is told when a box's clock is more than one minute from the hub's (R-1384, D79)
gates / gates (push) Successful in 2m11s
gates / gates (push) Successful in 2m11s
The host report's new sent_at (the box clock at the moment of sending) is compared with the hub's clock at arrival. Both directions; told once; clears within 30 s. A report that waited on the box raises nothing (reported_at is not read). A second box off in the same direction points at the hub's own clock. Operator-only. Not deployed. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
26f4f7660c
commit
88e636445a
9 files changed
+318
-2
No files matched your search
@@ -0,0 +1,139 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// clockReport is a host report of `host` whose agent stamped `sent_at` = the hub's now + skew, and whose collect
|
||||
// started `collectAge` before that (reported_at) — a report that WAITED on the box before it was sent.
|
||||
func clockReport(host string, skew, collectAge time.Duration, withSentAt bool) string {
|
||||
b := validReportBody(host)
|
||||
now := time.Now().Add(skew)
|
||||
extra := fmt.Sprintf(`,"reported_at":%q`, now.Add(-collectAge).UTC().Format(time.RFC3339))
|
||||
if withSentAt {
|
||||
extra += fmt.Sprintf(`,"sent_at":%q`, now.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
return b[:strings.LastIndex(b, "}")] + extra + "}"
|
||||
}
|
||||
|
||||
func clockSetup(t *testing.T) (*Handler, *store.Store, func(host string, skew, age time.Duration, sent bool)) {
|
||||
t.Helper()
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
|
||||
for _, id := range []string{"h1", "h2"} {
|
||||
st.UpsertHost(&store.Host{HostID: id, CustomerID: "c1", APIKey: "KEY-" + id})
|
||||
}
|
||||
post := func(host string, skew, age time.Duration, sent bool) {
|
||||
t.Helper()
|
||||
if rr := do(h, http.MethodPost, "/host-report", "KEY-"+host, clockReport(host, skew, age, sent)); rr.Code != 200 {
|
||||
t.Fatalf("report: %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
return h, st, post
|
||||
}
|
||||
|
||||
func clockEvents(t *testing.T, st *store.Store) (wrong, ok []store.Event) {
|
||||
t.Helper()
|
||||
since := time.Now().Add(-time.Hour)
|
||||
wrong, _ = st.GetEventsByType("c1", eventBoxClockWrong, since)
|
||||
ok, _ = st.GetEventsByType("c1", eventBoxClockOK, since)
|
||||
return
|
||||
}
|
||||
|
||||
// R-1384 (D79): a box more than one minute off, in EITHER direction, is told to the operator once; it clears once
|
||||
// when the clock is back. COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-box-clock.txt).
|
||||
func TestBoxClock_BothDirectionsOnceThenClear(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
skew time.Duration
|
||||
dir string
|
||||
}{{"behind", -95 * time.Second, "behind"}, {"ahead", 95 * time.Second, "ahead"}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, st, post := clockSetup(t)
|
||||
post("h1", 20*time.Second, 0, true) // in step: nothing
|
||||
post("h1", -45*time.Second, 0, true) // off, but under one minute: nothing
|
||||
if w, o := clockEvents(t, st); len(w) != 0 || len(o) != 0 {
|
||||
t.Fatalf("a clock under one minute off raised %d wrong, %d ok", len(w), len(o))
|
||||
}
|
||||
post("h1", c.skew, 0, true)
|
||||
post("h1", c.skew, 0, true)
|
||||
w, o := clockEvents(t, st)
|
||||
if len(w) != 1 || len(o) != 0 {
|
||||
t.Fatalf("wrong=%d ok=%d, want ONE alarm for two reports of the same wrong clock", len(w), len(o))
|
||||
}
|
||||
if w[0].Severity != "warning" || !strings.Contains(w[0].Message, "h1") || !strings.Contains(w[0].Message, "seconds "+c.dir+" ") ||
|
||||
!strings.Contains(string(w[0].DetailsJSON), `"direction":"`+c.dir+`"`) {
|
||||
t.Errorf("alarm = %+v, want a warning that names h1 and %q", w[0], c.dir)
|
||||
}
|
||||
if strings.Contains(w[0].Message, "HUB's own clock") {
|
||||
t.Errorf("one box alone must not point at the hub's clock: %s", w[0].Message)
|
||||
}
|
||||
post("h1", c.skew/2, 0, true) // 47 s: under the alarm line, above the clear line → no word
|
||||
if w, o := clockEvents(t, st); len(w) != 1 || len(o) != 0 {
|
||||
t.Fatalf("between the two lines: wrong=%d ok=%d, want 1 and 0", len(w), len(o))
|
||||
}
|
||||
post("h1", 2*time.Second, 0, true)
|
||||
post("h1", 2*time.Second, 0, true)
|
||||
w, o = clockEvents(t, st)
|
||||
if len(w) != 1 || len(o) != 1 || !strings.Contains(o[0].Message, "was "+c.dir) {
|
||||
t.Fatalf("after the clock came back: wrong=%d ok=%d (%v), want one recovery", len(w), len(o), o)
|
||||
}
|
||||
post("h1", c.skew, 0, true) // and a second time it goes wrong, it is told again
|
||||
if w, _ := clockEvents(t, st); len(w) != 2 {
|
||||
t.Fatalf("a second wrong clock after a recovery: wrong=%d, want 2", len(w))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The trap the ruling names: a report that WAITED on the box (a long collect) carries an old reported_at. That is not
|
||||
// a wrong clock, and it raises nothing — the hub reads the time stamped at SENDING. And an agent that sends no
|
||||
// sent_at is never judged, however old its reported_at is.
|
||||
func TestBoxClock_AQueuedReportRaisesNothing(t *testing.T) {
|
||||
_, st, post := clockSetup(t)
|
||||
post("h1", 0, 10*time.Minute, true) // collected 10 minutes ago, sent now, clock right
|
||||
post("h1", 0, 10*time.Minute, false) // an older agent: no sent_at at all
|
||||
post("h1", 0, -10*time.Minute, false)
|
||||
if w, o := clockEvents(t, st); len(w) != 0 || len(o) != 0 {
|
||||
t.Fatalf("a report that waited on the box raised %d wrong, %d ok — an old reported_at is not a wrong clock", len(w), len(o))
|
||||
}
|
||||
// positive control: the same handler, the same box, does alarm on the send time
|
||||
post("h1", -5*time.Minute, 0, true)
|
||||
if w, _ := clockEvents(t, st); len(w) != 1 {
|
||||
t.Fatalf("control: a send time 5 minutes behind raised %d alarm(s), want 1", len(w))
|
||||
}
|
||||
}
|
||||
|
||||
// When the HUB's clock is wrong, every healthy box looks wrong the same way. The second box's alarm says so. A box
|
||||
// off in the OTHER direction does not.
|
||||
func TestBoxClock_SecondBoxSameDirectionPointsAtTheHub(t *testing.T) {
|
||||
_, st, post := clockSetup(t)
|
||||
post("h1", 3*time.Minute, 0, true)
|
||||
post("h2", -3*time.Minute, 0, true)
|
||||
w, _ := clockEvents(t, st)
|
||||
if len(w) != 2 {
|
||||
t.Fatalf("wrong=%d, want 2", len(w))
|
||||
}
|
||||
for _, e := range w {
|
||||
if strings.Contains(e.Message, "HUB's own clock") {
|
||||
t.Errorf("boxes off in OPPOSITE directions must not point at the hub: %s", e.Message)
|
||||
}
|
||||
}
|
||||
post("h2", 3*time.Minute, 0, true) // h2 flips: now both are ahead
|
||||
w, _ = clockEvents(t, st)
|
||||
var last store.Event
|
||||
for _, e := range w {
|
||||
if strings.Contains(e.Message, "of h2 ") && strings.Contains(e.Message, "ahead") {
|
||||
last = e
|
||||
}
|
||||
}
|
||||
if len(w) != 3 || !strings.Contains(last.Message, "HUB's own clock") || !strings.Contains(last.Message, "h1") {
|
||||
t.Fatalf("wrong=%d; the second box ahead must name the hub's clock and h1: %q", len(w), last.Message)
|
||||
}
|
||||
}
|
||||
@@ -778,7 +778,11 @@ type hostReportPayload struct {
|
||||
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
|
||||
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
|
||||
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
|
||||
Guests []struct {
|
||||
// SentAt (R-1384, D79): the box's clock at the moment the agent SENDS the report (stamped in the agent's
|
||||
// Client.Report, after the collect). "" = an agent that does not stamp it → the clock is not judged.
|
||||
// reported_at is NOT used for this: it is stamped when the collect starts, and a collect can take long.
|
||||
SentAt string `json:"sent_at"`
|
||||
Guests []struct {
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
@@ -942,6 +946,7 @@ type hostStorageTarget struct {
|
||||
// handleHostReport ingests the agent's host-report (the heartbeat) and returns the
|
||||
// control envelope (agent spec §5).
|
||||
func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
received := time.Now() // before the body is read: the clock check (R-1384) compares the box's send time with THIS
|
||||
hostID, custID, isGlobal, ok := h.checkAuthHost(r)
|
||||
if !ok {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
@@ -1012,6 +1017,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||||
h.checkWGTunnel(hostID, custID, rep.Wireguard)
|
||||
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
|
||||
h.checkBoxClock(hostID, custID, rep.SentAt, received)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
@@ -3240,6 +3246,91 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Box clock alarm (R-1384, D79, `09` §3 decision 300; `08-alarm-ladder.md`). The hub compares the box's clock at
|
||||
// the moment of SENDING (`sent_at`) with its own clock at the moment the request arrives. More than one minute apart,
|
||||
// in either direction → one operator-only `box_clock_wrong`; back within 30 seconds → one `box_clock_ok` (the gap
|
||||
// between 30 and 60 s keeps a clock at the edge from mailing at every report).
|
||||
//
|
||||
// What this can NOT tell: whose clock is wrong. A slow line can only make a box look BEHIND, and by less than the
|
||||
// agent's request timeout (30 s), so it cannot raise the alarm alone. When the HUB's clock is wrong, every healthy box
|
||||
// looks wrong in the SAME direction — so the message says so when a second box is off in the same direction.
|
||||
// A report with no `sent_at` (an older agent) is never judged. Pinned by TestBoxClock_*.
|
||||
const (
|
||||
eventBoxClockWrong = "box_clock_wrong"
|
||||
eventBoxClockOK = "box_clock_ok"
|
||||
boxClockRaiseAbove = time.Minute
|
||||
boxClockClearBelow = 30 * time.Second
|
||||
)
|
||||
|
||||
func (h *Handler) checkBoxClock(hostID, custID, sentAt string, received time.Time) {
|
||||
if sentAt == "" {
|
||||
return
|
||||
}
|
||||
sent, err := time.Parse(time.RFC3339Nano, sentAt)
|
||||
if err != nil {
|
||||
h.logger.Printf("[WARN] host %s: sent_at is not a time (%d bytes); the clock is not judged", hostID, len(sentAt))
|
||||
return
|
||||
}
|
||||
skew := sent.Sub(received) // > 0: the box is AHEAD of the hub
|
||||
abs := skew
|
||||
if abs < 0 {
|
||||
abs = -abs
|
||||
}
|
||||
dir := "ahead"
|
||||
if skew < 0 {
|
||||
dir = "behind"
|
||||
}
|
||||
key := func(d string) string { return "box_clock:" + hostID + ":" + d }
|
||||
emit := func(typ, sev, msg string, details map[string]any) {
|
||||
dj, _ := json.Marshal(details)
|
||||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||||
}
|
||||
}
|
||||
raised := ""
|
||||
for _, d := range []string{"ahead", "behind"} {
|
||||
if !h.store.OSAlarmRaised(key(d)).IsZero() {
|
||||
raised = d
|
||||
}
|
||||
}
|
||||
secs := int64(abs / time.Second)
|
||||
switch {
|
||||
case abs > boxClockRaiseAbove:
|
||||
if raised == dir {
|
||||
return // said once
|
||||
}
|
||||
if raised != "" {
|
||||
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
|
||||
}
|
||||
_ = h.store.SetOSAlarmRaised(key(dir), received)
|
||||
// the hub's own clock: another box off in the same direction points at the hub, not at the boxes
|
||||
var same []string
|
||||
if hosts, lerr := h.store.ListHosts(); lerr == nil {
|
||||
for _, o := range hosts {
|
||||
if o.HostID != hostID && !h.store.OSAlarmRaised("box_clock:"+o.HostID+":"+dir).IsZero() {
|
||||
same = append(same, o.HostID)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(same)
|
||||
hint := ""
|
||||
if len(same) > 0 {
|
||||
hint = fmt.Sprintf(" %d other box(es) are %s too (%s): check the HUB's own clock (DooPlex) first.", len(same), dir, strings.Join(same, ", "))
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s clock: %d s %s of the hub (same direction: %d other)", hostID, secs, dir, len(same))
|
||||
emit(eventBoxClockWrong, "warning",
|
||||
fmt.Sprintf("The clock of %s is %d seconds %s the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time. Read its time service (`chronyc tracking`).%s", hostID, secs, map[string]string{"ahead": "ahead of", "behind": "behind"}[dir], hint),
|
||||
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "direction": dir, "same_direction": same})
|
||||
case abs <= boxClockClearBelow && raised != "":
|
||||
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
|
||||
h.logger.Printf("[INFO] host %s clock: in step again (%d s; was %s)", hostID, secs, raised)
|
||||
emit(eventBoxClockOK, "info", fmt.Sprintf("The clock of %s is in step with the hub's clock again (was %s).", hostID, raised),
|
||||
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "was": raised})
|
||||
}
|
||||
}
|
||||
|
||||
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
|
||||
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
|
||||
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,
|
||||
|
||||
Reference in new issue
Block a user