hub(D53): held apps reach the operator — mirror, Held apps list (host + Hosts page), reminder after the grace
gates / gates (push) Successful in 1m58s
gates / gates (push) Successful in 1m58s
app_held / app_hold_cleared allow-listed and operator-only; handleEvent mirrors them in held_apps; HeldAppsChecker sends ONE app_held_reminder (warning, operator-only, per-app cooldown) after the grace per reason (6 h / 1 d / 3 d), then weekly; recorded before sending; stale rows not reminded; the hub never clears a hold. Test-only alerting.held_app_grace_override. TestD53_* in api/monitor/notify/web, red-proved by six mutations. Commit only.
This commit is contained in:
1 parent
ecd82abab8
commit
7e57e824a2
27 files changed
+935
-2
No files matched your search
@@ -2262,6 +2262,12 @@ var allowedEventTypes = map[string]bool{
|
||||
// v0.123.0 (decision 28, controller v0.269.0): the box stopped an app that kept crashing / ran out
|
||||
// of memory. Household + operator; hu AND en `mail.event.*`; per-app cooldown on both legs.
|
||||
"app_stopped_unhealthy": true,
|
||||
// D53 (`08` §10, controller with internal/holdreport): the box's held apps — app_held for each hold in place (at once,
|
||||
// then daily), app_hold_cleared when one is gone. Severity info: never mailed on receipt; the hub mirrors them in
|
||||
// held_apps (handleEvent → recordHeldApp) and the reminder checker mails the OPERATOR after a grace. Operator-only
|
||||
// (notify.operatorOnlyEvents); hu AND en `mail.event.*` lines exist only because the coupling rule needs them.
|
||||
"app_held": true,
|
||||
"app_hold_cleared": true,
|
||||
|
||||
// R-871 (v0.134.0, controller v0.295.0, `09` decision 109): the box was off at its backup time and made the
|
||||
// missed backups when it came back. Info — recorded on the box's timeline, never mailed.
|
||||
@@ -2520,6 +2526,12 @@ func (h *Handler) handleEvent(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
h.logger.Printf("[INFO] Event from %s: %s (%s) — %s", payload.CustomerID, payload.EventType, payload.Severity, payload.Message)
|
||||
|
||||
// D53: mirror the box's held apps. A failure is logged and the event still dispatched — the box repeats
|
||||
// app_held daily, so the next one heals the mirror.
|
||||
if payload.EventType == "app_held" || payload.EventType == "app_hold_cleared" {
|
||||
h.recordHeldApp(payload.CustomerID, payload.EventType, payload.Details)
|
||||
}
|
||||
|
||||
// Dispatch notifications (non-blocking)
|
||||
if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessBoxEvent(payload.CustomerID, payload.EventType, payload.Severity,
|
||||
@@ -3189,3 +3201,40 @@ func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
|
||||
// clears a hold on a box; it only forgets one the box says is gone.
|
||||
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {
|
||||
var d struct {
|
||||
StackName string `json:"stack_name"`
|
||||
Reason string `json:"reason"`
|
||||
Since string `json:"since"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &d); err != nil || d.StackName == "" {
|
||||
h.logger.Printf("[WARN] [held-apps] %s from %s has no readable stack_name — not recorded", eventType, customerID)
|
||||
return
|
||||
}
|
||||
if eventType == "app_hold_cleared" {
|
||||
had, err := h.store.ClearHeldApp(customerID, d.StackName)
|
||||
if err != nil {
|
||||
h.logger.Printf("[WARN] [held-apps] clear %s/%s: %v", customerID, d.StackName, err)
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[INFO] [held-apps] %s/%s no longer held (a row existed: %v)", customerID, d.StackName, had)
|
||||
return
|
||||
}
|
||||
var since time.Time
|
||||
if d.Since != "" {
|
||||
if t, err := time.Parse(time.RFC3339, d.Since); err == nil {
|
||||
since = t
|
||||
} else {
|
||||
h.logger.Printf("[WARN] [held-apps] %s/%s: since %q is not RFC3339 — the hub's first sighting is the clock", customerID, d.StackName, d.Since)
|
||||
}
|
||||
}
|
||||
if err := h.store.UpsertHeldApp(customerID, d.StackName, d.Reason, since, d.Why, time.Now()); err != nil {
|
||||
h.logger.Printf("[WARN] [held-apps] record %s/%s: %v", customerID, d.StackName, err)
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[DEBUG] [held-apps] %s/%s held (%s since %s)", customerID, d.StackName, d.Reason, d.Since)
|
||||
}
|
||||
Reference in new issue
Block a user