hub (unreleased): the operator is told why a box's agent does not trust Proxmox (proxmox_trust stanza → proxmox_trust_problem / _ok, operator-only; R-1396)
gates / gates (push) Successful in 3m35s

This commit is contained in:
admin committed 2026-11-14 16:52:00 +01:00
1 parent d809a1d444
commit 26f4f7660c
8 files changed
+255 -4

No files matched your search

+103 -1
View File
@@ -775,7 +775,10 @@ type hostReportPayload struct {
} `json:"host"`
// Wireguard (R-1204): the box's own view of its tunnel to ep0. nil = an agent with no tunnel stanza.
Wireguard *hostWireguard `json:"wireguard"`
Guests []struct {
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
Guests []struct {
VMID int `json:"vmid"`
Name string `json:"name"`
Status string `json:"status"`
@@ -1008,6 +1011,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
h.checkWGTunnel(hostID, custID, rep.Wireguard)
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
h.checkCrash(hostID, custID, body)
h.checkDeliberateOff(hostID, custID, &rep)
@@ -3236,6 +3240,104 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
}
}
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,
// cause) — raised once, not at every report — and one `proxmox_trust_ok` when the box reports no problem again.
// A box with NO stanza (an older agent) never alarms and never clears. Pinned by TestProxmoxTrust_*.
const (
eventProxmoxTrustProblem = "proxmox_trust_problem"
eventProxmoxTrustOK = "proxmox_trust_ok"
)
type hostProxmoxTrust struct {
Mode string `json:"mode"`
Problem string `json:"problem"`
Detail string `json:"detail"`
CANotAfter string `json:"ca_not_after"`
CADaysLeft *int `json:"ca_days_left"`
MovedAt string `json:"moved_at"`
}
// proxmoxTrustProblems is the CLOSED list of causes the hub alarms on (the agent's internal/pvetrust and
// proxmox.Trust* names). A cause outside it is logged and stored under "other" — never dropped, never trusted as text.
var proxmoxTrustProblems = map[string]string{
"pinned_cert_changed": "the pinned certificate is not the one Proxmox serves any more",
"ca_mismatch": "the root CA copy did not sign the certificate Proxmox serves",
"cert_expired": "the certificate Proxmox serves, or its root CA, has ended",
"ca_file_unreadable": "the root CA copy is missing or holds no certificate",
"name_mismatch": "the certificate Proxmox serves does not name the address the agent calls",
"ca_copy_unavailable": "the box's root step gave no copy of Proxmox's root CA",
"ca_copy_does_not_verify": "a call to Proxmox with the root CA copy alone failed; the box keeps its pinned certificate",
"ca_copy_ends_soon": "the root CA copy ends within 30 days",
"config_not_written": "the agent could not save its new trust settings; the box keeps its pinned certificate",
}
func (h *Handler) checkProxmoxTrust(hostID, custID string, t *hostProxmoxTrust) {
if t == nil {
return
}
now := time.Now()
emit := func(typ, sev, msg string, details map[string]any) {
dj, _ := json.Marshal(details)
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
}
}
if t.Problem == "" {
// the box reports no problem: clear every cause that was raised for it, with ONE recovery event
var cleared []string
for cause := range proxmoxTrustProblems {
key := "pve_trust:" + hostID + ":" + cause
if !h.store.OSAlarmRaised(key).IsZero() {
_ = h.store.SetOSAlarmRaised(key, time.Time{})
cleared = append(cleared, cause)
}
}
if key := "pve_trust:" + hostID + ":other"; !h.store.OSAlarmRaised(key).IsZero() {
_ = h.store.SetOSAlarmRaised(key, time.Time{})
cleared = append(cleared, "other")
}
if len(cleared) > 0 {
sort.Strings(cleared)
h.logger.Printf("[INFO] host %s proxmox trust: no problem any more (was: %s; mode %s)", hostID, strings.Join(cleared, ", "), t.Mode)
emit(eventProxmoxTrustOK, "info", fmt.Sprintf("The agent of %s trusts Proxmox again (mode: %s). Was: %s.", hostID, t.Mode, strings.Join(cleared, ", ")),
map[string]any{"host_id": hostID, "mode": t.Mode, "was": cleared})
}
return
}
cause, text := t.Problem, proxmoxTrustProblems[t.Problem]
if text == "" {
cause, text = "other", "a cause this hub does not know"
}
key := "pve_trust:" + hostID + ":" + cause
if !h.store.OSAlarmRaised(key).IsZero() {
return // said once; the box repeats it at every report
}
_ = h.store.SetOSAlarmRaised(key, now)
sev := "error"
if cause == "ca_copy_ends_soon" {
sev = "warning"
}
days := ""
if t.CADaysLeft != nil {
days = fmt.Sprintf(" The root CA copy ends in %d day(s) (%s).", *t.CADaysLeft, t.CANotAfter)
}
h.logger.Printf("[WARN] host %s proxmox trust: %s (mode %s)", hostID, cause, t.Mode)
emit(eventProxmoxTrustProblem, sev, fmt.Sprintf("Trust in Proxmox on %s: %s (mode: %s).%s Design: `03` §12a.", hostID, text, t.Mode, days),
map[string]any{"host_id": hostID, "cause": cause, "mode": t.Mode, "reported_cause": truncateRunes(t.Problem, 60), "ca_not_after": t.CANotAfter})
}
func truncateRunes(s string, n int) string {
r := []rune(s)
if len(r) <= n {
return s
}
return string(r[:n])
}
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
// clears a hold on a box; it only forgets one the box says is gone.
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {