hub 0.164.0 (unreleased): host_swap_short / host_swap_ok operator alarm from the agent's host and guest swap numbers (R-1192 follow-up); wire gate allowlists guests.spec.swap_bytes
gates / gates (push) Failing after 14m53s
gates / gates (push) Failing after 14m53s
This commit is contained in:
@@ -765,6 +765,10 @@ type hostReportPayload struct {
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
MemoryPercent float64 `json:"memory_percent"`
|
||||
DiskPercent float64 `json:"disk_percent"`
|
||||
// R-1192 follow-up (agent ≥ the swap release): the host's swap and its guests' swap allowance, MB. nil = an
|
||||
// older agent → not evaluated.
|
||||
HostSwapMB *int64 `json:"host_swap_mb"`
|
||||
GuestSwapMB *int64 `json:"guest_swap_mb"`
|
||||
} `json:"host"`
|
||||
Guests []struct {
|
||||
VMID int `json:"vmid"`
|
||||
@@ -987,6 +991,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
h.carryEscrow(custID)
|
||||
}
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// R-1192 follow-up (night 2026-10-21). The guest swap fix (agent: every guest may swap as much as its RAM) moves a
|
||||
// memory burst onto the HOST's swap. Our installer gives an 8 GB machine 7.75 GB of it (measured on demo-hp); a host
|
||||
// with none — a BYO install, or swap turned off — gets nothing from the fix, and the household's first big upload can
|
||||
// freeze the box again. The agent reports both numbers; the hub tells the OPERATOR once per host when the host's swap
|
||||
// is below its guests' allowance (host_swap_short), and once when it is covered again (host_swap_ok). Edge-triggered
|
||||
// from the host's own newest swap event, so a hub restart does not mail again. Operator-only (notify.operatorOnlyEvents):
|
||||
// the household cannot add swap to its host.
|
||||
const (
|
||||
eventHostSwapShort = "host_swap_short"
|
||||
eventHostSwapOK = "host_swap_ok"
|
||||
)
|
||||
|
||||
func (h *Handler) checkHostSwap(hostID, custID string, hostMB, guestMB *int64) {
|
||||
if hostMB == nil || guestMB == nil || custID == "" {
|
||||
return // an older agent: not evaluated
|
||||
}
|
||||
short := *guestMB > 0 && *hostMB < *guestMB
|
||||
prev, err := h.store.LatestHostSwapEvent(custID, hostID)
|
||||
if err != nil {
|
||||
h.logger.Printf("[WARN] host %s swap check: reading the last swap event failed: %v", hostID, err)
|
||||
return
|
||||
}
|
||||
wasShort := prev != nil && prev.EventType == eventHostSwapShort
|
||||
var typ, sev, msg string
|
||||
switch {
|
||||
case short && !wasShort:
|
||||
typ, sev = eventHostSwapShort, "warning"
|
||||
msg = fmt.Sprintf("Host %s: its swap (%d MB) is below its guests' swap allowance (%d MB). The guest swap fix needs swap on the host behind it; "+
|
||||
"without it a memory burst — a big first upload on an 8 GB box — can freeze the box for minutes (R-1192). Add swap on the host "+
|
||||
"(our installer gives an 8 GB machine 7.75 GB) or lower the guest's RAM. The household cannot fix this.", hostID, *hostMB, *guestMB)
|
||||
case !short && wasShort:
|
||||
typ, sev = eventHostSwapOK, "info"
|
||||
msg = fmt.Sprintf("Host %s: its swap (%d MB) covers its guests' swap allowance (%d MB) again.", hostID, *hostMB, *guestMB)
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s: %s (host swap %d MB, guests' swap %d MB)", hostID, typ, *hostMB, *guestMB)
|
||||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "host_swap_mb": *hostMB, "guest_swap_mb": *guestMB})
|
||||
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
|
||||
return // the audit row first: a mail without its event row lies
|
||||
}
|
||||
if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// R-1192 follow-up (night 2026-10-21): a host whose swap is below its guests' swap allowance raises host_swap_short
|
||||
// ONCE (edge-triggered from the host's newest swap event, so repeats and a hub restart stay quiet), host_swap_ok once
|
||||
// when covered, and an older agent (no numbers) is not evaluated. Operator-only (TestR1192_HostSwapEventsAreOperatorOnly
|
||||
// in notify).
|
||||
// RED-PROOF: with checkHostSwap not called from handleHostReport the first assertion fails (0 events).
|
||||
func swapReportBody(hostID string, hostMB, guestMB int64) string {
|
||||
b := validReportBody(hostID)
|
||||
return strings.Replace(b, `"uptime_seconds":100}`, `"uptime_seconds":100,"host_swap_mb":`+strconv.FormatInt(hostMB, 10)+`,"guest_swap_mb":`+strconv.FormatInt(guestMB, 10)+`}`, 1)
|
||||
}
|
||||
|
||||
func TestR1192_HostSwapShortIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
post := func(body string) {
|
||||
t.Helper()
|
||||
if rr := do(h, http.MethodPost, "/host-report", "HKEY", body); rr.Code != http.StatusOK {
|
||||
t.Fatalf("host-report = %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
count := func(typ string) int {
|
||||
evs, _ := st.GetEventsByType("c1", typ, time.Time{})
|
||||
return len(evs)
|
||||
}
|
||||
post(validReportBody("h1")) // an older agent: no numbers → nothing
|
||||
if count(eventHostSwapShort) != 0 {
|
||||
t.Fatal("an agent that reports no swap numbers must not be evaluated")
|
||||
}
|
||||
post(swapReportBody("h1", 0, 4096)) // no host swap, a 4 GB guest allowance
|
||||
post(swapReportBody("h1", 0, 4096))
|
||||
if n := count(eventHostSwapShort); n != 1 {
|
||||
t.Fatalf("host_swap_short events = %d, want exactly 1 (edge-triggered)", n)
|
||||
}
|
||||
evs, _ := st.GetEventsByType("c1", eventHostSwapShort, time.Time{})
|
||||
if !strings.Contains(evs[0].Message, "Host h1:") || !strings.Contains(evs[0].DetailsJSON, `"guest_swap_mb":4096`) {
|
||||
t.Fatalf("the alarm must name the host and both numbers: %q %s", evs[0].Message, evs[0].DetailsJSON)
|
||||
}
|
||||
post(swapReportBody("h1", 7935, 4096)) // the installer's 8 GB host
|
||||
post(swapReportBody("h1", 7935, 4096))
|
||||
if n := count(eventHostSwapOK); n != 1 {
|
||||
t.Fatalf("host_swap_ok events = %d, want exactly 1", n)
|
||||
}
|
||||
post(swapReportBody("h1", 7935, 4096))
|
||||
if count(eventHostSwapShort) != 1 {
|
||||
t.Fatal("a covered host must not alarm again")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user