hub 0.164.0 (unreleased): host_swap_short / host_swap_ok operator alarm from the agent's host and guest swap numbers (R-1192 follow-up); wire gate allowlists guests.spec.swap_bytes
gates / gates (push) Failing after 14m53s
gates / gates (push) Failing after 14m53s
This commit is contained in:
@@ -1,3 +1,13 @@
|
||||
## v0.164.0 (unreleased — built the night of 2026-10-21, NOT deployed; the deploy is the operator's morning yes)
|
||||
|
||||
- **R-1192 follow-up — `host_swap_short` (operator-only).** The agent's guest swap fix (swap = RAM) moves a memory burst
|
||||
onto the host's swap; a host with none gets nothing from it. Agents from the same release report `host.host_swap_mb`
|
||||
and `host.guest_swap_mb`; when the first is below the second the hub mails the operator ONCE per host
|
||||
(`host_swap_short`, warning, both numbers), and once when it is covered again (`host_swap_ok`, info) — edge-triggered
|
||||
from the host's newest swap event (`store.LatestHostSwapEvent`, by `details.host_id`), so repeats and a hub restart
|
||||
stay quiet. An older agent (no numbers) is not evaluated. Both in `notify.operatorOnlyEvents`. Tests
|
||||
`TestR1192_HostSwapShortIsRaisedOnceAndClearedOnce`, `TestR1192_HostSwapEventsAreOperatorOnly`.
|
||||
|
||||
## v0.163.0 — a rebuilt box gets the household's sealed key package back (R-1176); one run per daily job on clock-change days (R-1183); abandoning one set-aside copy keeps the other copies' packages (R-1182); the „new link" page tells a household its old box is still linked (R-1203) (2026-10-21)
|
||||
|
||||
Built for the operator-attended deploy of 2026-10-21 (rulings D24 and the R-1176 shape, same day).
|
||||
|
||||
@@ -765,6 +765,10 @@ type hostReportPayload struct {
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
MemoryPercent float64 `json:"memory_percent"`
|
||||
DiskPercent float64 `json:"disk_percent"`
|
||||
// R-1192 follow-up (agent ≥ the swap release): the host's swap and its guests' swap allowance, MB. nil = an
|
||||
// older agent → not evaluated.
|
||||
HostSwapMB *int64 `json:"host_swap_mb"`
|
||||
GuestSwapMB *int64 `json:"guest_swap_mb"`
|
||||
} `json:"host"`
|
||||
Guests []struct {
|
||||
VMID int `json:"vmid"`
|
||||
@@ -987,6 +991,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
h.carryEscrow(custID)
|
||||
}
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// R-1192 follow-up (night 2026-10-21). The guest swap fix (agent: every guest may swap as much as its RAM) moves a
|
||||
// memory burst onto the HOST's swap. Our installer gives an 8 GB machine 7.75 GB of it (measured on demo-hp); a host
|
||||
// with none — a BYO install, or swap turned off — gets nothing from the fix, and the household's first big upload can
|
||||
// freeze the box again. The agent reports both numbers; the hub tells the OPERATOR once per host when the host's swap
|
||||
// is below its guests' allowance (host_swap_short), and once when it is covered again (host_swap_ok). Edge-triggered
|
||||
// from the host's own newest swap event, so a hub restart does not mail again. Operator-only (notify.operatorOnlyEvents):
|
||||
// the household cannot add swap to its host.
|
||||
const (
|
||||
eventHostSwapShort = "host_swap_short"
|
||||
eventHostSwapOK = "host_swap_ok"
|
||||
)
|
||||
|
||||
func (h *Handler) checkHostSwap(hostID, custID string, hostMB, guestMB *int64) {
|
||||
if hostMB == nil || guestMB == nil || custID == "" {
|
||||
return // an older agent: not evaluated
|
||||
}
|
||||
short := *guestMB > 0 && *hostMB < *guestMB
|
||||
prev, err := h.store.LatestHostSwapEvent(custID, hostID)
|
||||
if err != nil {
|
||||
h.logger.Printf("[WARN] host %s swap check: reading the last swap event failed: %v", hostID, err)
|
||||
return
|
||||
}
|
||||
wasShort := prev != nil && prev.EventType == eventHostSwapShort
|
||||
var typ, sev, msg string
|
||||
switch {
|
||||
case short && !wasShort:
|
||||
typ, sev = eventHostSwapShort, "warning"
|
||||
msg = fmt.Sprintf("Host %s: its swap (%d MB) is below its guests' swap allowance (%d MB). The guest swap fix needs swap on the host behind it; "+
|
||||
"without it a memory burst — a big first upload on an 8 GB box — can freeze the box for minutes (R-1192). Add swap on the host "+
|
||||
"(our installer gives an 8 GB machine 7.75 GB) or lower the guest's RAM. The household cannot fix this.", hostID, *hostMB, *guestMB)
|
||||
case !short && wasShort:
|
||||
typ, sev = eventHostSwapOK, "info"
|
||||
msg = fmt.Sprintf("Host %s: its swap (%d MB) covers its guests' swap allowance (%d MB) again.", hostID, *hostMB, *guestMB)
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s: %s (host swap %d MB, guests' swap %d MB)", hostID, typ, *hostMB, *guestMB)
|
||||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "host_swap_mb": *hostMB, "guest_swap_mb": *guestMB})
|
||||
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
|
||||
return // the audit row first: a mail without its event row lies
|
||||
}
|
||||
if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// R-1192 follow-up (night 2026-10-21): a host whose swap is below its guests' swap allowance raises host_swap_short
|
||||
// ONCE (edge-triggered from the host's newest swap event, so repeats and a hub restart stay quiet), host_swap_ok once
|
||||
// when covered, and an older agent (no numbers) is not evaluated. Operator-only (TestR1192_HostSwapEventsAreOperatorOnly
|
||||
// in notify).
|
||||
// RED-PROOF: with checkHostSwap not called from handleHostReport the first assertion fails (0 events).
|
||||
func swapReportBody(hostID string, hostMB, guestMB int64) string {
|
||||
b := validReportBody(hostID)
|
||||
return strings.Replace(b, `"uptime_seconds":100}`, `"uptime_seconds":100,"host_swap_mb":`+strconv.FormatInt(hostMB, 10)+`,"guest_swap_mb":`+strconv.FormatInt(guestMB, 10)+`}`, 1)
|
||||
}
|
||||
|
||||
func TestR1192_HostSwapShortIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
post := func(body string) {
|
||||
t.Helper()
|
||||
if rr := do(h, http.MethodPost, "/host-report", "HKEY", body); rr.Code != http.StatusOK {
|
||||
t.Fatalf("host-report = %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
count := func(typ string) int {
|
||||
evs, _ := st.GetEventsByType("c1", typ, time.Time{})
|
||||
return len(evs)
|
||||
}
|
||||
post(validReportBody("h1")) // an older agent: no numbers → nothing
|
||||
if count(eventHostSwapShort) != 0 {
|
||||
t.Fatal("an agent that reports no swap numbers must not be evaluated")
|
||||
}
|
||||
post(swapReportBody("h1", 0, 4096)) // no host swap, a 4 GB guest allowance
|
||||
post(swapReportBody("h1", 0, 4096))
|
||||
if n := count(eventHostSwapShort); n != 1 {
|
||||
t.Fatalf("host_swap_short events = %d, want exactly 1 (edge-triggered)", n)
|
||||
}
|
||||
evs, _ := st.GetEventsByType("c1", eventHostSwapShort, time.Time{})
|
||||
if !strings.Contains(evs[0].Message, "Host h1:") || !strings.Contains(evs[0].DetailsJSON, `"guest_swap_mb":4096`) {
|
||||
t.Fatalf("the alarm must name the host and both numbers: %q %s", evs[0].Message, evs[0].DetailsJSON)
|
||||
}
|
||||
post(swapReportBody("h1", 7935, 4096)) // the installer's 8 GB host
|
||||
post(swapReportBody("h1", 7935, 4096))
|
||||
if n := count(eventHostSwapOK); n != 1 {
|
||||
t.Fatalf("host_swap_ok events = %d, want exactly 1", n)
|
||||
}
|
||||
post(swapReportBody("h1", 7935, 4096))
|
||||
if count(eventHostSwapShort) != 1 {
|
||||
t.Fatal("a covered host must not alarm again")
|
||||
}
|
||||
}
|
||||
@@ -854,6 +854,10 @@ var operatorOnlyEvents = map[string]bool{
|
||||
// Registered in the same commit that mints the types.
|
||||
"escrow_carried": true,
|
||||
"escrow_carry_refused": true,
|
||||
// R-1192 follow-up (night 2026-10-21): the host's swap is below its guests' swap allowance — and covered again.
|
||||
// The household cannot add swap to its host; the operator can.
|
||||
"host_swap_short": true,
|
||||
"host_swap_ok": true,
|
||||
// R-1203 (2026-10-21): a household asked for a new box link while its old host is still linked — the operator
|
||||
// gets the host name to delete. Registered in the same commit that mints the type.
|
||||
"bind_old_host_linked": true,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package notify
|
||||
|
||||
import "testing"
|
||||
|
||||
// R-1192 follow-up (night 2026-10-21): the household cannot add swap to its host — the host-swap alarm and its
|
||||
// recovery are the operator's only.
|
||||
func TestR1192_HostSwapEventsAreOperatorOnly(t *testing.T) {
|
||||
for _, e := range []string{"host_swap_short", "host_swap_ok"} {
|
||||
if !operatorOnlyEvents[e] {
|
||||
t.Fatalf("%s must be operator-only", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2736,6 +2736,28 @@ func (s *Store) LatestHostLivenessEvent(customerID, hostID string) (*Event, erro
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
// LatestHostSwapEvent returns the newest host_swap_short / host_swap_ok event for hostID, or nil (R-1192 follow-up).
|
||||
// The host is read from the event's details (host_id), so a customer with two hosts keeps two states.
|
||||
func (s *Store) LatestHostSwapEvent(customerID, hostID string) (*Event, error) {
|
||||
var e Event
|
||||
var createdAt string
|
||||
err := s.db.QueryRow(`
|
||||
SELECT id, customer_id, event_type, severity, message, details_json, source, created_at
|
||||
FROM events
|
||||
WHERE customer_id = ? AND event_type IN ('host_swap_short', 'host_swap_ok') AND json_extract(details_json, '$.host_id') = ?
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1`, customerID, hostID,
|
||||
).Scan(&e.ID, &e.CustomerID, &e.EventType, &e.Severity, &e.Message, &e.DetailsJSON, &e.Source, &createdAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.CreatedAt = parseSQLiteTime(createdAt)
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
// GetAllRecentEvents returns the most recent events across all customers.
|
||||
func (s *Store) GetAllRecentEvents(limit int) ([]Event, error) {
|
||||
rows, err := s.db.Query(`
|
||||
|
||||
@@ -205,6 +205,9 @@ ALLOWLIST = {
|
||||
"decodes vmid/name/status/controller_version from the guest list and nothing more."),
|
||||
(_AH, "guests.spec.memory_bytes"): (
|
||||
"redundant: as guests.spec.disk_bytes — sizing is hub-owned intent, not mirrored reality."),
|
||||
(_AH, "guests.spec.swap_bytes"): (
|
||||
"redundant: the hub consults the host's SUM, host.guest_swap_mb (host_swap_short, R-1192 follow-up, night "
|
||||
"2026-10-21); the per-guest number stays in the stored raw report for the operator."),
|
||||
(_AH, "storage_targets.smart.model_name"): (
|
||||
"redundant for a verdict: the hub decodes smart.health plus every counter it bands on. The "
|
||||
"model string is a display label with no threshold attached to it."),
|
||||
|
||||
Reference in New Issue
Block a user