hub (unreleased): four silent failures tell the operator (R-1383, D78)
gates / gates (push) Successful in 2m9s

A hub daily check skipped while the hub was down; an operator mail that was never sent; a signed job a box did not
take; a signed job a box refused, and the guest-power watchdog's give-up (both from the agent). Operator-only.
No new table. Not deployed.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
admin committed 2026-11-14 17:29:53 +01:00
1 parent 88e636445a
commit 1886030782
22 files changed
+1269 -2

No files matched your search

+126
View File
@@ -12,6 +12,7 @@ import (
"io"
"log"
"net/http"
"regexp"
"sort"
"strings"
"sync"
@@ -778,6 +779,9 @@ type hostReportPayload struct {
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
// GuestPowerGaveUp (R-1383, D78): the guests the agent's guest-power watchdog stopped trying to start (three
// failed starts). nil/empty = none, or an agent that does not report it.
GuestPowerGaveUp []hostGuestPowerGaveUp `json:"guest_power_gave_up"`
// SentAt (R-1384, D79): the box's clock at the moment the agent SENDS the report (stamped in the agent's
// Client.Report, after the collect). "" = an agent that does not stamp it → the clock is not judged.
// reported_at is NOT used for this: it is stamped when the collect starts, and a collect can take long.
@@ -1018,6 +1022,13 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
h.checkWGTunnel(hostID, custID, rep.Wireguard)
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
h.checkBoxClock(hostID, custID, rep.SentAt, received)
runningGuests := map[int]bool{}
for _, g := range rep.Guests {
if g.Status == "running" {
runningGuests[g.VMID] = true
}
}
h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests)
h.checkCrash(hostID, custID, body)
h.checkDeliberateOff(hostID, custID, &rep)
@@ -2072,6 +2083,7 @@ func (h *Handler) handleDeleteJob(w http.ResponseWriter, r *http.Request, pathHo
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op"))
h.logger.Printf("[INFO] host %s cleared signed-op job %s (executed or rejected)", pathHostID, jobID)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
@@ -3246,6 +3258,120 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
}
}
// The four silent failures, the two a BOX tells (R-1383, D78, `09` §3 decision 299; `08-alarm-ladder.md`).
//
// (a) signed_job_refused — the agent clears a signed job from its queue when it ran it AND when it refused it; until
// now the hub could not tell the two apart („executed or rejected”), so an agent update that was refused left no
// trace outside the box's journal (clock list CA09-4, CA09-7). An agent of release 3 says how the job ended in the
// DELETE's query: outcome = done | rejected | failed | malformed, a reason from a CLOSED list, and the op's name.
// Anything but `done` → one operator-only event. No outcome (an older agent) → nothing. The box's words never reach
// a mail: an unknown reason is told as „other”, an op name is cut to [a-z0-9_].
// (b) guest_power_gave_up — see checkGuestPowerGaveUp.
// Pinned by TestSignedJobOutcome_* and TestGuestPowerGaveUp_*.
const (
eventSignedJobRefused = "signed_job_refused"
eventGuestPowerGaveUp = "guest_power_gave_up"
eventGuestPowerRunning = "guest_power_ok"
)
// signedJobReasons is the closed list of the agent's reasons (signedjobs.reasonCode), each with the operator's
// sentence. The two clock reasons name the clock, because a box with a wrong clock refuses EVERY job.
var signedJobReasons = map[string]string{
"expired": "its validity had run out when the box read it. If the job was fresh, the box's clock is AHEAD",
"not_yet_valid": "it was not valid yet by the box's clock. The box's clock is BEHIND",
"window_too_long": "its validity window is longer than the box allows",
"replay": "the box saw this job's number before (a job runs once)",
"bad_signature": "the signature did not verify",
"unknown_signer": "the signing key is not one the box trusts",
"target": "it names another box",
"namespace": "it was signed for another use",
"malformed": "the box could not read it",
"role_denied": "the signing key may not sign this kind of job",
"binding_mismatch": "it is signed for another action or device",
"execution_failed": "it was accepted and its run FAILED on the box",
}
var signedJobOpRe = regexp.MustCompile(`^[a-z0-9_]{1,40}$`)
func (h *Handler) noteSignedJobOutcome(hostID, jobID, outcome, reason, op string) {
switch outcome {
case "rejected", "failed", "malformed":
default:
return // "done", or an agent that does not say
}
key := "signed_job_refused:" + hostID + ":" + jobID
if !h.store.OSAlarmRaised(key).IsZero() {
return // the agent repeats a DELETE whose answer it did not get
}
_ = h.store.SetOSAlarmRaised(key, time.Now())
text := signedJobReasons[reason]
if text == "" {
reason, text = "other", "the box gave a reason this hub does not know"
}
if !signedJobOpRe.MatchString(op) {
op = "unknown"
}
custID := ""
if host, err := h.store.GetHost(hostID); err == nil && host != nil {
custID = host.CustomerID
}
what := "REFUSED"
if outcome == "failed" {
what = "ran and FAILED"
}
h.logger.Printf("[WARN] host %s signed job %s (%s): %s — %s", hostID, jobID, op, outcome, reason)
msg := fmt.Sprintf("The signed job %s (%s) for %s was %s: %s. The job is gone from the queue and will not run again. Do: read the cause, then sign a new job.", jobID, op, hostID, what, text)
details, _ := json.Marshal(map[string]any{"host_id": hostID, "job_id": jobID, "op": op, "outcome": outcome, "reason": reason})
if _, err := h.store.SaveEvent(custID, eventSignedJobRefused, "error", msg, string(details), "hub"); err != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", eventSignedJobRefused, hostID, err)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, eventSignedJobRefused, "error", msg, string(details), "hub")
}
}
type hostGuestPowerGaveUp struct {
VMID int `json:"vmid"`
Since string `json:"since"`
Attempts int `json:"attempts"`
}
// checkGuestPowerGaveUp: the agent's guest-power watchdog starts a guest that must run and is stopped; after three
// failed starts it stops trying, and until now it wrote one line in the host's journal and told nobody (clock list
// CA07-7) — the household's apps are off and the only outside sign was „the box does not report”, which is not even
// true here (the HOST reports). One operator-only `guest_power_gave_up` (error) per guest, told once for one down
// period; one `guest_power_ok` when a later report shows that guest RUNNING. The agent's record is memory: after an
// agent restart the entry is gone while the guest is still down — so „gone from the list” alone never clears.
func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running map[int]bool) {
emit := func(typ, sev, msg string, details map[string]any) {
dj, _ := json.Marshal(details)
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
}
}
key := func(vmid int) string { return fmt.Sprintf("guest_power:%s:%d", hostID, vmid) }
for _, g := range gave {
if g.VMID <= 0 || running[g.VMID] || !h.store.OSAlarmRaised(key(g.VMID)).IsZero() {
continue
}
_ = h.store.SetOSAlarmRaised(key(g.VMID), time.Now())
h.logger.Printf("[WARN] host %s: the guest-power watchdog gave up on guest %d (%d attempts)", hostID, g.VMID, g.Attempts)
emit(eventGuestPowerGaveUp, "error",
fmt.Sprintf("Guest %d on %s is STOPPED and the agent gave up starting it after %d failed attempts. The household's apps are off. The agent does not try again by itself. Do: read `journalctl -u felhom-agent` on the box for „guest-power”, start the guest, and read why it did not start.", g.VMID, hostID, g.Attempts),
map[string]any{"host_id": hostID, "vmid": g.VMID, "attempts": g.Attempts, "since": g.Since})
}
for vmid := range running {
if h.store.OSAlarmRaised(key(vmid)).IsZero() {
continue
}
_ = h.store.SetOSAlarmRaised(key(vmid), time.Time{})
h.logger.Printf("[INFO] host %s: guest %d runs again (the guest-power watchdog had given up)", hostID, vmid)
emit(eventGuestPowerRunning, "info", fmt.Sprintf("Guest %d on %s runs again.", vmid, hostID),
map[string]any{"host_id": hostID, "vmid": vmid})
}
}
// Box clock alarm (R-1384, D79, `09` §3 decision 300; `08-alarm-ladder.md`). The hub compares the box's clock at
// the moment of SENDING (`sent_at`) with its own clock at the moment the request arrives. More than one minute apart,
// in either direction → one operator-only `box_clock_wrong`; back within 30 seconds → one `box_clock_ok` (the gap