hub(D53): held apps reach the operator — mirror, Held apps list (host + Hosts page), reminder after the grace
gates / gates (push) Successful in 1m58s

app_held / app_hold_cleared allow-listed and operator-only; handleEvent mirrors them in held_apps; HeldAppsChecker
sends ONE app_held_reminder (warning, operator-only, per-app cooldown) after the grace per reason (6 h / 1 d / 3 d),
then weekly; recorded before sending; stale rows not reminded; the hub never clears a hold. Test-only
alerting.held_app_grace_override. TestD53_* in api/monitor/notify/web, red-proved by six mutations. Commit only.
This commit is contained in:
admin committed 2026-10-26 18:25:00 +01:00
1 parent ecd82abab8
commit 7e57e824a2
27 files changed
+935 -2

No files matched your search

+49
View File
@@ -2262,6 +2262,12 @@ var allowedEventTypes = map[string]bool{
// v0.123.0 (decision 28, controller v0.269.0): the box stopped an app that kept crashing / ran out
// of memory. Household + operator; hu AND en `mail.event.*`; per-app cooldown on both legs.
"app_stopped_unhealthy": true,
// D53 (`08` §10, controller with internal/holdreport): the box's held apps — app_held for each hold in place (at once,
// then daily), app_hold_cleared when one is gone. Severity info: never mailed on receipt; the hub mirrors them in
// held_apps (handleEvent → recordHeldApp) and the reminder checker mails the OPERATOR after a grace. Operator-only
// (notify.operatorOnlyEvents); hu AND en `mail.event.*` lines exist only because the coupling rule needs them.
"app_held": true,
"app_hold_cleared": true,
// R-871 (v0.134.0, controller v0.295.0, `09` decision 109): the box was off at its backup time and made the
// missed backups when it came back. Info — recorded on the box's timeline, never mailed.
@@ -2520,6 +2526,12 @@ func (h *Handler) handleEvent(w http.ResponseWriter, r *http.Request) {
h.logger.Printf("[INFO] Event from %s: %s (%s) — %s", payload.CustomerID, payload.EventType, payload.Severity, payload.Message)
// D53: mirror the box's held apps. A failure is logged and the event still dispatched — the box repeats
// app_held daily, so the next one heals the mirror.
if payload.EventType == "app_held" || payload.EventType == "app_hold_cleared" {
h.recordHeldApp(payload.CustomerID, payload.EventType, payload.Details)
}
// Dispatch notifications (non-blocking)
if h.dispatcher != nil {
go h.dispatcher.ProcessBoxEvent(payload.CustomerID, payload.EventType, payload.Severity,
@@ -3189,3 +3201,40 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
}
}
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
// clears a hold on a box; it only forgets one the box says is gone.
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {
var d struct {
StackName string `json:"stack_name"`
Reason string `json:"reason"`
Since string `json:"since"`
Why string `json:"why"`
}
if err := json.Unmarshal(raw, &d); err != nil || d.StackName == "" {
h.logger.Printf("[WARN] [held-apps] %s from %s has no readable stack_name — not recorded", eventType, customerID)
return
}
if eventType == "app_hold_cleared" {
had, err := h.store.ClearHeldApp(customerID, d.StackName)
if err != nil {
h.logger.Printf("[WARN] [held-apps] clear %s/%s: %v", customerID, d.StackName, err)
return
}
h.logger.Printf("[INFO] [held-apps] %s/%s no longer held (a row existed: %v)", customerID, d.StackName, had)
return
}
var since time.Time
if d.Since != "" {
if t, err := time.Parse(time.RFC3339, d.Since); err == nil {
since = t
} else {
h.logger.Printf("[WARN] [held-apps] %s/%s: since %q is not RFC3339 — the hub's first sighting is the clock", customerID, d.StackName, d.Since)
}
}
if err := h.store.UpsertHeldApp(customerID, d.StackName, d.Reason, since, d.Why, time.Now()); err != nil {
h.logger.Printf("[WARN] [held-apps] record %s/%s: %v", customerID, d.StackName, err)
return
}
h.logger.Printf("[DEBUG] [held-apps] %s/%s held (%s since %s)", customerID, d.StackName, d.Reason, d.Since)
}