hub (unreleased): a box cannot mint operator events — only a queued job is told as refused, the give-up list is bounded (R-1383); docs for the clock-step fix and HSTS (R-1395, R-1050)
gates / gates (push) Successful in 2m3s
gates / gates (push) Successful in 2m3s
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
1886030782
commit
186f3fca58
11 files changed
+184
-14
No files matched your search
+33
-10
@@ -1028,7 +1028,11 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
runningGuests[g.VMID] = true
|
||||
}
|
||||
}
|
||||
h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests)
|
||||
knownGuests := map[int]bool{}
|
||||
for _, g := range rep.Guests {
|
||||
knownGuests[g.VMID] = true
|
||||
}
|
||||
h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests, knownGuests)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
@@ -2078,12 +2082,17 @@ func (h *Handler) handleDeleteJob(w http.ResponseWriter, r *http.Request, pathHo
|
||||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteSignedJob(pathHostID, jobID); err != nil {
|
||||
removed, err := h.store.DeleteSignedJobN(pathHostID, jobID)
|
||||
if err != nil {
|
||||
h.logger.Printf("[ERROR] delete job %s for %s: %v", jobID, pathHostID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op"))
|
||||
if removed > 0 {
|
||||
// only a job that WAS in this box's queue can be told as refused: the id comes from the box, and a box must
|
||||
// not mint operator events by clearing ids it invents. It also makes a repeated DELETE silent.
|
||||
h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op"))
|
||||
}
|
||||
h.logger.Printf("[INFO] host %s cleared signed-op job %s (executed or rejected)", pathHostID, jobID)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -3264,7 +3273,7 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
// now the hub could not tell the two apart („executed or rejected”), so an agent update that was refused left no
|
||||
// trace outside the box's journal (clock list CA09-4, CA09-7). An agent of release 3 says how the job ended in the
|
||||
// DELETE's query: outcome = done | rejected | failed | malformed, a reason from a CLOSED list, and the op's name.
|
||||
// Anything but `done` → one operator-only event. No outcome (an older agent) → nothing. The box's words never reach
|
||||
// Anything but `done` → one operator-only event, and ONLY for a job that was in that box's queue. No outcome (an older agent) → nothing. The box's words never reach
|
||||
// a mail: an unknown reason is told as „other”, an op name is cut to [a-z0-9_].
|
||||
// (b) guest_power_gave_up — see checkGuestPowerGaveUp.
|
||||
// Pinned by TestSignedJobOutcome_* and TestGuestPowerGaveUp_*.
|
||||
@@ -3293,17 +3302,24 @@ var signedJobReasons = map[string]string{
|
||||
|
||||
var signedJobOpRe = regexp.MustCompile(`^[a-z0-9_]{1,40}$`)
|
||||
|
||||
// signedJobIDRe is the alphabet of a job id the hub mints (hex) or the operator's tool names; anything else is not
|
||||
// quoted in a mail.
|
||||
var signedJobIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`)
|
||||
|
||||
// guestPowerMaxEntries bounds the give-up list read from ONE report (a box has a handful of guests).
|
||||
const guestPowerMaxEntries = 16
|
||||
|
||||
func (h *Handler) noteSignedJobOutcome(hostID, jobID, outcome, reason, op string) {
|
||||
switch outcome {
|
||||
case "rejected", "failed", "malformed":
|
||||
default:
|
||||
return // "done", or an agent that does not say
|
||||
}
|
||||
key := "signed_job_refused:" + hostID + ":" + jobID
|
||||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||||
return // the agent repeats a DELETE whose answer it did not get
|
||||
if !signedJobIDRe.MatchString(jobID) {
|
||||
// the id was a row of the hub's own queue (the caller checked), but it is put there by the operator's upload
|
||||
// as free text — an id outside the hub's own alphabet is named, not quoted.
|
||||
jobID = "(id not shown)"
|
||||
}
|
||||
_ = h.store.SetOSAlarmRaised(key, time.Now())
|
||||
text := signedJobReasons[reason]
|
||||
if text == "" {
|
||||
reason, text = "other", "the box gave a reason this hub does not know"
|
||||
@@ -3341,7 +3357,14 @@ type hostGuestPowerGaveUp struct {
|
||||
// true here (the HOST reports). One operator-only `guest_power_gave_up` (error) per guest, told once for one down
|
||||
// period; one `guest_power_ok` when a later report shows that guest RUNNING. The agent's record is memory: after an
|
||||
// agent restart the entry is gone while the guest is still down — so „gone from the list” alone never clears.
|
||||
func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running map[int]bool) {
|
||||
//
|
||||
// The list comes from the box, so it is bounded: only a guest that the SAME report lists (and not as running) is
|
||||
// read, and at most guestPowerMaxEntries entries — a box cannot mint an alarm per invented vmid.
|
||||
func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running, known map[int]bool) {
|
||||
if len(gave) > guestPowerMaxEntries {
|
||||
h.logger.Printf("[WARN] host %s: guest_power_gave_up lists %d guests — only the first %d are read", hostID, len(gave), guestPowerMaxEntries)
|
||||
gave = gave[:guestPowerMaxEntries]
|
||||
}
|
||||
emit := func(typ, sev, msg string, details map[string]any) {
|
||||
dj, _ := json.Marshal(details)
|
||||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||||
@@ -3352,7 +3375,7 @@ func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestP
|
||||
}
|
||||
key := func(vmid int) string { return fmt.Sprintf("guest_power:%s:%d", hostID, vmid) }
|
||||
for _, g := range gave {
|
||||
if g.VMID <= 0 || running[g.VMID] || !h.store.OSAlarmRaised(key(g.VMID)).IsZero() {
|
||||
if g.VMID <= 0 || !known[g.VMID] || running[g.VMID] || !h.store.OSAlarmRaised(key(g.VMID)).IsZero() {
|
||||
continue
|
||||
}
|
||||
_ = h.store.SetOSAlarmRaised(key(g.VMID), time.Now())
|
||||
|
||||
@@ -59,6 +59,15 @@ func TestSignedJobOutcome_RefusedIsToldDoneIsNot(t *testing.T) {
|
||||
if evs[0].Severity != "error" || !strings.Contains(evs[0].DetailsJSON, `"reason":"expired"`) {
|
||||
t.Errorf("event = %+v", evs[0])
|
||||
}
|
||||
// a box „clears” job ids that were never in its queue: no event — a box cannot mint operator mails
|
||||
for _, id := range []string{"never-queued", "never-queued-2", "x%0Ay"} {
|
||||
if rr := do(h, http.MethodDelete, "/hosts/h1/jobs/"+id+"?outcome=rejected&reason=expired&op=agent_update", "HKEY", ""); rr.Code != 200 {
|
||||
t.Fatalf("DELETE of an unknown job: %d", rr.Code)
|
||||
}
|
||||
}
|
||||
if evs := r1383Events(t, st, eventSignedJobRefused); len(evs) != 1 {
|
||||
t.Fatalf("clearing ids that were never queued raised events: %d, want still 1", len(evs))
|
||||
}
|
||||
// a failed run, and words the hub does not know: told, and the box's own text never reaches the mail
|
||||
del("j-fail", "?outcome=failed&reason=execution_failed&op=agent_config_update")
|
||||
del("j-odd", "?outcome=rejected&reason=%3Cscript%3E&op=%3Cb%3EX")
|
||||
@@ -133,4 +142,18 @@ func TestGuestPowerGaveUp_ToldOnceClearedOnlyByARunningGuest(t *testing.T) {
|
||||
if c := counts(); c != "gave_up=2 ok=1" {
|
||||
t.Fatalf("a new down period: %s", c)
|
||||
}
|
||||
// the list is the box's: a vmid the same report does not list is not read, and a long list is cut
|
||||
many := "["
|
||||
for i := 0; i < 500; i++ {
|
||||
many += fmt.Sprintf(`{"vmid":%d,"since":"2026-11-14T02:00:00Z","attempts":3},`, 5000+i)
|
||||
}
|
||||
many += `{"vmid":101,"since":"2026-11-14T02:00:00Z","attempts":3}]` // 101 IS a guest of the report (stopped), but entry 501
|
||||
post("stopped", many)
|
||||
if c := counts(); c != "gave_up=2 ok=1" {
|
||||
t.Fatalf("500 invented guests and a list past the cap: %s, want no new alarm", c)
|
||||
}
|
||||
post("stopped", `[{"vmid":101,"since":"2026-11-14T02:00:00Z","attempts":3},{"vmid":7777,"since":"x","attempts":3}]`)
|
||||
if c := counts(); c != "gave_up=3 ok=1" {
|
||||
t.Fatalf("a real second guest (101) within the cap: %s, want one more alarm and none for vmid 7777", c)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user