hub (unreleased): the operator is told why a box's agent does not trust Proxmox (proxmox_trust stanza → proxmox_trust_problem / _ok, operator-only; R-1396)
gates / gates (push) Successful in 3m35s
gates / gates (push) Successful in 3m35s
This commit is contained in:
1 parent
d809a1d444
commit
26f4f7660c
8 files changed
+255
-4
No files matched your search
+103
-1
@@ -775,7 +775,10 @@ type hostReportPayload struct {
|
||||
} `json:"host"`
|
||||
// Wireguard (R-1204): the box's own view of its tunnel to ep0. nil = an agent with no tunnel stanza.
|
||||
Wireguard *hostWireguard `json:"wireguard"`
|
||||
Guests []struct {
|
||||
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
|
||||
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
|
||||
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
|
||||
Guests []struct {
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
@@ -1008,6 +1011,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||||
h.checkWGTunnel(hostID, custID, rep.Wireguard)
|
||||
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
h.checkDeliberateOff(hostID, custID, &rep)
|
||||
|
||||
@@ -3236,6 +3240,104 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
|
||||
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
|
||||
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,
|
||||
// cause) — raised once, not at every report — and one `proxmox_trust_ok` when the box reports no problem again.
|
||||
// A box with NO stanza (an older agent) never alarms and never clears. Pinned by TestProxmoxTrust_*.
|
||||
const (
|
||||
eventProxmoxTrustProblem = "proxmox_trust_problem"
|
||||
eventProxmoxTrustOK = "proxmox_trust_ok"
|
||||
)
|
||||
|
||||
type hostProxmoxTrust struct {
|
||||
Mode string `json:"mode"`
|
||||
Problem string `json:"problem"`
|
||||
Detail string `json:"detail"`
|
||||
CANotAfter string `json:"ca_not_after"`
|
||||
CADaysLeft *int `json:"ca_days_left"`
|
||||
MovedAt string `json:"moved_at"`
|
||||
}
|
||||
|
||||
// proxmoxTrustProblems is the CLOSED list of causes the hub alarms on (the agent's internal/pvetrust and
|
||||
// proxmox.Trust* names). A cause outside it is logged and stored under "other" — never dropped, never trusted as text.
|
||||
var proxmoxTrustProblems = map[string]string{
|
||||
"pinned_cert_changed": "the pinned certificate is not the one Proxmox serves any more",
|
||||
"ca_mismatch": "the root CA copy did not sign the certificate Proxmox serves",
|
||||
"cert_expired": "the certificate Proxmox serves, or its root CA, has ended",
|
||||
"ca_file_unreadable": "the root CA copy is missing or holds no certificate",
|
||||
"name_mismatch": "the certificate Proxmox serves does not name the address the agent calls",
|
||||
"ca_copy_unavailable": "the box's root step gave no copy of Proxmox's root CA",
|
||||
"ca_copy_does_not_verify": "a call to Proxmox with the root CA copy alone failed; the box keeps its pinned certificate",
|
||||
"ca_copy_ends_soon": "the root CA copy ends within 30 days",
|
||||
"config_not_written": "the agent could not save its new trust settings; the box keeps its pinned certificate",
|
||||
}
|
||||
|
||||
func (h *Handler) checkProxmoxTrust(hostID, custID string, t *hostProxmoxTrust) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
emit := func(typ, sev, msg string, details map[string]any) {
|
||||
dj, _ := json.Marshal(details)
|
||||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||||
}
|
||||
}
|
||||
if t.Problem == "" {
|
||||
// the box reports no problem: clear every cause that was raised for it, with ONE recovery event
|
||||
var cleared []string
|
||||
for cause := range proxmoxTrustProblems {
|
||||
key := "pve_trust:" + hostID + ":" + cause
|
||||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||||
cleared = append(cleared, cause)
|
||||
}
|
||||
}
|
||||
if key := "pve_trust:" + hostID + ":other"; !h.store.OSAlarmRaised(key).IsZero() {
|
||||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||||
cleared = append(cleared, "other")
|
||||
}
|
||||
if len(cleared) > 0 {
|
||||
sort.Strings(cleared)
|
||||
h.logger.Printf("[INFO] host %s proxmox trust: no problem any more (was: %s; mode %s)", hostID, strings.Join(cleared, ", "), t.Mode)
|
||||
emit(eventProxmoxTrustOK, "info", fmt.Sprintf("The agent of %s trusts Proxmox again (mode: %s). Was: %s.", hostID, t.Mode, strings.Join(cleared, ", ")),
|
||||
map[string]any{"host_id": hostID, "mode": t.Mode, "was": cleared})
|
||||
}
|
||||
return
|
||||
}
|
||||
cause, text := t.Problem, proxmoxTrustProblems[t.Problem]
|
||||
if text == "" {
|
||||
cause, text = "other", "a cause this hub does not know"
|
||||
}
|
||||
key := "pve_trust:" + hostID + ":" + cause
|
||||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||||
return // said once; the box repeats it at every report
|
||||
}
|
||||
_ = h.store.SetOSAlarmRaised(key, now)
|
||||
sev := "error"
|
||||
if cause == "ca_copy_ends_soon" {
|
||||
sev = "warning"
|
||||
}
|
||||
days := ""
|
||||
if t.CADaysLeft != nil {
|
||||
days = fmt.Sprintf(" The root CA copy ends in %d day(s) (%s).", *t.CADaysLeft, t.CANotAfter)
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s proxmox trust: %s (mode %s)", hostID, cause, t.Mode)
|
||||
emit(eventProxmoxTrustProblem, sev, fmt.Sprintf("Trust in Proxmox on %s: %s (mode: %s).%s Design: `03` §12a.", hostID, text, t.Mode, days),
|
||||
map[string]any{"host_id": hostID, "cause": cause, "mode": t.Mode, "reported_cause": truncateRunes(t.Problem, 60), "ca_not_after": t.CANotAfter})
|
||||
}
|
||||
|
||||
func truncateRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n])
|
||||
}
|
||||
|
||||
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
|
||||
// clears a hold on a box; it only forgets one the box says is gone.
|
||||
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
func trustReport(stanza string) string {
|
||||
if stanza == "-" {
|
||||
return validReportBody("h1")
|
||||
}
|
||||
b := validReportBody("h1")
|
||||
return b[:strings.LastIndex(b, "}")] + `,"proxmox_trust":` + stanza + "}"
|
||||
}
|
||||
|
||||
func trustSetup(t *testing.T) (*Handler, *store.Store) {
|
||||
t.Helper()
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
return h, st
|
||||
}
|
||||
|
||||
func trustCounts(t *testing.T, st *store.Store) (problems, oks []store.Event) {
|
||||
t.Helper()
|
||||
since := time.Now().Add(-time.Hour)
|
||||
problems, _ = st.GetEventsByType("c1", eventProxmoxTrustProblem, since)
|
||||
oks, _ = st.GetEventsByType("c1", eventProxmoxTrustOK, since)
|
||||
return
|
||||
}
|
||||
|
||||
// R-1396: a cause is told ONCE (the box repeats it at every report), a second cause is told too, and one recovery
|
||||
// event follows when the box reports no problem. A healthy stanza and an agent without the stanza raise nothing.
|
||||
// COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub.txt).
|
||||
func TestProxmoxTrust_CauseOnceThenRecovery(t *testing.T) {
|
||||
h, st := trustSetup(t)
|
||||
post := func(stanza string) {
|
||||
t.Helper()
|
||||
if rr := do(h, http.MethodPost, "/host-report", "HKEY", trustReport(stanza)); rr.Code != 200 {
|
||||
t.Fatalf("report: %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
post(`{"mode":"ca"}`)
|
||||
post("-")
|
||||
if p, o := trustCounts(t, st); len(p) != 0 || len(o) != 0 {
|
||||
t.Fatalf("a healthy box raised %d problem(s), %d ok", len(p), len(o))
|
||||
}
|
||||
bad := `{"mode":"fingerprint","problem":"ca_copy_does_not_verify","detail":"<script>x</script>","ca_days_left":3560,"ca_not_after":"2036-08-17T15:44:42Z"}`
|
||||
post(bad)
|
||||
post(bad)
|
||||
post(bad)
|
||||
p, o := trustCounts(t, st)
|
||||
if len(p) != 1 || len(o) != 0 {
|
||||
t.Fatalf("problems=%d ok=%d, want ONE problem for three reports of the same cause", len(p), len(o))
|
||||
}
|
||||
if p[0].Severity != "error" || !strings.Contains(p[0].Message, "keeps its pinned certificate") || !strings.Contains(p[0].Message, "h1") {
|
||||
t.Fatalf("event = %+v", p[0])
|
||||
}
|
||||
if strings.Contains(p[0].Message, "<script>") || strings.Contains(p[0].DetailsJSON, "<script>") {
|
||||
t.Fatalf("the box's free text reached the operator's mail: %+v", p[0])
|
||||
}
|
||||
post(`{"mode":"ca","problem":"ca_copy_ends_soon","ca_days_left":12,"ca_not_after":"2026-11-26T00:00:00Z"}`)
|
||||
p, _ = trustCounts(t, st)
|
||||
if len(p) != 2 {
|
||||
t.Fatalf("a second cause was not told: %d", len(p))
|
||||
}
|
||||
var soon store.Event
|
||||
for _, e := range p {
|
||||
if strings.Contains(e.Message, "ends within 30 days") {
|
||||
soon = e
|
||||
}
|
||||
}
|
||||
if soon.Severity != "warning" || !strings.Contains(soon.Message, "12 day") {
|
||||
t.Fatalf("ends-soon event = %+v", soon)
|
||||
}
|
||||
post(`{"mode":"ca"}`)
|
||||
post(`{"mode":"ca"}`)
|
||||
p, o = trustCounts(t, st)
|
||||
if len(p) != 2 || len(o) != 1 {
|
||||
t.Fatalf("problems=%d ok=%d, want ONE recovery for both causes", len(p), len(o))
|
||||
}
|
||||
// after the recovery the same cause is a NEW problem
|
||||
post(bad)
|
||||
if p, _ = trustCounts(t, st); len(p) != 3 {
|
||||
t.Fatalf("the cause came back and was not told again: %d", len(p))
|
||||
}
|
||||
// an agent WITHOUT the stanza neither clears nor raises
|
||||
post("-")
|
||||
if _, o = trustCounts(t, st); len(o) != 1 {
|
||||
t.Fatalf("a report with no stanza cleared the problem: ok=%d", len(o))
|
||||
}
|
||||
}
|
||||
|
||||
// A cause the hub does not know is told as "other" — once — and its text is not passed on.
|
||||
func TestProxmoxTrust_UnknownCause(t *testing.T) {
|
||||
h, st := trustSetup(t)
|
||||
for i := 0; i < 2; i++ {
|
||||
do(h, http.MethodPost, "/host-report", "HKEY", trustReport(`{"mode":"ca","problem":"Kedves Ügyfél, kattints ide","detail":"x"}`))
|
||||
}
|
||||
p, _ := trustCounts(t, st)
|
||||
if len(p) != 1 || !strings.Contains(p[0].Message, "a cause this hub does not know") || strings.Contains(p[0].Message, "Kedves") {
|
||||
t.Fatalf("events = %+v", p)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user