hub (unreleased): the operator is told why a box's agent does not trust Proxmox (proxmox_trust stanza → proxmox_trust_problem / _ok, operator-only; R-1396)
gates / gates (push) Successful in 3m35s
gates / gates (push) Successful in 3m35s
This commit is contained in:
1 parent
d809a1d444
commit
26f4f7660c
8 files changed
+255
-4
No files matched your search
+103
-1
@@ -775,7 +775,10 @@ type hostReportPayload struct {
|
||||
} `json:"host"`
|
||||
// Wireguard (R-1204): the box's own view of its tunnel to ep0. nil = an agent with no tunnel stanza.
|
||||
Wireguard *hostWireguard `json:"wireguard"`
|
||||
Guests []struct {
|
||||
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
|
||||
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
|
||||
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
|
||||
Guests []struct {
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
@@ -1008,6 +1011,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||||
h.checkWGTunnel(hostID, custID, rep.Wireguard)
|
||||
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
@@ -3236,6 +3240,104 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
|
||||
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
|
||||
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,
|
||||
// cause) — raised once, not at every report — and one `proxmox_trust_ok` when the box reports no problem again.
|
||||
// A box with NO stanza (an older agent) never alarms and never clears. Pinned by TestProxmoxTrust_*.
|
||||
const (
|
||||
eventProxmoxTrustProblem = "proxmox_trust_problem"
|
||||
eventProxmoxTrustOK = "proxmox_trust_ok"
|
||||
)
|
||||
|
||||
type hostProxmoxTrust struct {
|
||||
Mode string `json:"mode"`
|
||||
Problem string `json:"problem"`
|
||||
Detail string `json:"detail"`
|
||||
CANotAfter string `json:"ca_not_after"`
|
||||
CADaysLeft *int `json:"ca_days_left"`
|
||||
MovedAt string `json:"moved_at"`
|
||||
}
|
||||
|
||||
// proxmoxTrustProblems is the CLOSED list of causes the hub alarms on (the agent's internal/pvetrust and
|
||||
// proxmox.Trust* names). A cause outside it is logged and stored under "other" — never dropped, never trusted as text.
|
||||
var proxmoxTrustProblems = map[string]string{
|
||||
"pinned_cert_changed": "the pinned certificate is not the one Proxmox serves any more",
|
||||
"ca_mismatch": "the root CA copy did not sign the certificate Proxmox serves",
|
||||
"cert_expired": "the certificate Proxmox serves, or its root CA, has ended",
|
||||
"ca_file_unreadable": "the root CA copy is missing or holds no certificate",
|
||||
"name_mismatch": "the certificate Proxmox serves does not name the address the agent calls",
|
||||
"ca_copy_unavailable": "the box's root step gave no copy of Proxmox's root CA",
|
||||
"ca_copy_does_not_verify": "a call to Proxmox with the root CA copy alone failed; the box keeps its pinned certificate",
|
||||
"ca_copy_ends_soon": "the root CA copy ends within 30 days",
|
||||
"config_not_written": "the agent could not save its new trust settings; the box keeps its pinned certificate",
|
||||
}
|
||||
|
||||
func (h *Handler) checkProxmoxTrust(hostID, custID string, t *hostProxmoxTrust) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
emit := func(typ, sev, msg string, details map[string]any) {
|
||||
dj, _ := json.Marshal(details)
|
||||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||||
}
|
||||
}
|
||||
if t.Problem == "" {
|
||||
// the box reports no problem: clear every cause that was raised for it, with ONE recovery event
|
||||
var cleared []string
|
||||
for cause := range proxmoxTrustProblems {
|
||||
key := "pve_trust:" + hostID + ":" + cause
|
||||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||||
cleared = append(cleared, cause)
|
||||
}
|
||||
}
|
||||
if key := "pve_trust:" + hostID + ":other"; !h.store.OSAlarmRaised(key).IsZero() {
|
||||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||||
cleared = append(cleared, "other")
|
||||
}
|
||||
if len(cleared) > 0 {
|
||||
sort.Strings(cleared)
|
||||
h.logger.Printf("[INFO] host %s proxmox trust: no problem any more (was: %s; mode %s)", hostID, strings.Join(cleared, ", "), t.Mode)
|
||||
emit(eventProxmoxTrustOK, "info", fmt.Sprintf("The agent of %s trusts Proxmox again (mode: %s). Was: %s.", hostID, t.Mode, strings.Join(cleared, ", ")),
|
||||
map[string]any{"host_id": hostID, "mode": t.Mode, "was": cleared})
|
||||
}
|
||||
return
|
||||
}
|
||||
cause, text := t.Problem, proxmoxTrustProblems[t.Problem]
|
||||
if text == "" {
|
||||
cause, text = "other", "a cause this hub does not know"
|
||||
}
|
||||
key := "pve_trust:" + hostID + ":" + cause
|
||||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||||
return // said once; the box repeats it at every report
|
||||
}
|
||||
_ = h.store.SetOSAlarmRaised(key, now)
|
||||
sev := "error"
|
||||
if cause == "ca_copy_ends_soon" {
|
||||
sev = "warning"
|
||||
}
|
||||
days := ""
|
||||
if t.CADaysLeft != nil {
|
||||
days = fmt.Sprintf(" The root CA copy ends in %d day(s) (%s).", *t.CADaysLeft, t.CANotAfter)
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s proxmox trust: %s (mode %s)", hostID, cause, t.Mode)
|
||||
emit(eventProxmoxTrustProblem, sev, fmt.Sprintf("Trust in Proxmox on %s: %s (mode: %s).%s Design: `03` §12a.", hostID, text, t.Mode, days),
|
||||
map[string]any{"host_id": hostID, "cause": cause, "mode": t.Mode, "reported_cause": truncateRunes(t.Problem, 60), "ca_not_after": t.CANotAfter})
|
||||
}
|
||||
|
||||
func truncateRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n])
|
||||
}
|
||||
|
||||
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
|
||||
// clears a hold on a box; it only forgets one the box says is gone.
|
||||
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {
|
||||
|
||||
Reference in new issue
Block a user