controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
This commit is contained in:
@@ -37,6 +37,11 @@ type AlertManager struct {
|
||||
// is health-report-driven). nil = channel up. It is prepended in GetAlerts so a dead controller→
|
||||
// agent link (disk/storage UI broken) is always visible regardless of the health-report cycle.
|
||||
agentChannelAlert *Alert
|
||||
// deadAppAlerts (fix-3, CAMPAIGN-3) is set each cycle by the health loop from the deployed-app
|
||||
// running-state view (which is NOT in the health report — it comes from the stack manager). Same
|
||||
// out-of-band, state-based, self-clearing model as agentChannelAlert: passing an empty slice when
|
||||
// every deployed app is running clears the banner with no manual dismissal.
|
||||
deadAppAlerts []Alert
|
||||
}
|
||||
|
||||
// NewAlertManager creates a new AlertManager.
|
||||
@@ -72,6 +77,63 @@ func (am *AlertManager) SetAgentChannelAlert(down bool, msg string) {
|
||||
}
|
||||
}
|
||||
|
||||
// DeadApp is a deployed app the health loop found not-running (fix-3). State is the container-state
|
||||
// string for the display (e.g. "stopped"/"exited").
|
||||
type DeadApp struct {
|
||||
Name string
|
||||
DisplayName string
|
||||
State string
|
||||
}
|
||||
|
||||
// deadAppGroupThreshold: above this many dead apps, collapse to ONE grouped alert (a reboot storm
|
||||
// with many NAS apps down should not paper the dashboard with a wall of banners — fix-3).
|
||||
const deadAppGroupThreshold = 3
|
||||
|
||||
// buildDeadAppAlerts turns the dead-app list into dashboard alerts (WARN). ≤ threshold → one per app;
|
||||
// more → a single grouped alert. Pure → unit-tested. Empty list → nil (clears the banner).
|
||||
func buildDeadAppAlerts(dead []DeadApp) []Alert {
|
||||
if len(dead) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(dead) > deadAppGroupThreshold {
|
||||
return []Alert{{
|
||||
ID: "deadapp-group",
|
||||
Level: "warning",
|
||||
Message: fmt.Sprintf("%d telepített alkalmazás nem fut — nézze meg a rendszermonitort", len(dead)),
|
||||
Link: "/monitoring",
|
||||
LinkText: "Rendszermonitor",
|
||||
}}
|
||||
}
|
||||
alerts := make([]Alert, 0, len(dead))
|
||||
for _, d := range dead {
|
||||
name := d.DisplayName
|
||||
if name == "" {
|
||||
name = d.Name
|
||||
}
|
||||
msg := "Telepített alkalmazás nem fut: " + name
|
||||
if d.State != "" {
|
||||
msg += " (" + d.State + ")"
|
||||
}
|
||||
alerts = append(alerts, Alert{
|
||||
ID: "deadapp-" + simpleHash(d.Name),
|
||||
Level: "warning",
|
||||
Message: msg,
|
||||
Link: "/monitoring",
|
||||
LinkText: "Rendszermonitor",
|
||||
})
|
||||
}
|
||||
return alerts
|
||||
}
|
||||
|
||||
// SetDeadAppAlerts stores the current deployed-not-running banner set (fix-3). State-based: called
|
||||
// each health cycle with the live dead-app list (empty clears it). Included in GetAlerts.
|
||||
func (am *AlertManager) SetDeadAppAlerts(dead []DeadApp) {
|
||||
alerts := buildDeadAppAlerts(dead)
|
||||
am.mu.Lock()
|
||||
am.deadAppAlerts = alerts
|
||||
am.mu.Unlock()
|
||||
}
|
||||
|
||||
// Refresh regenerates alerts from the latest health check report and config state.
|
||||
// Called after each health check cycle (every 5 minutes) and on storage state changes.
|
||||
func (am *AlertManager) Refresh(report *monitor.HealthReport, cfg *config.Config, backupMgr *backup.Manager, updateAvailable bool, latestVersion string, storagePaths ...[]settings.StoragePath) {
|
||||
@@ -182,7 +244,7 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
|
||||
am.mu.RLock()
|
||||
defer am.mu.RUnlock()
|
||||
|
||||
if len(am.alerts) == 0 && am.agentChannelAlert == nil {
|
||||
if len(am.alerts) == 0 && am.agentChannelAlert == nil && len(am.deadAppAlerts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -196,6 +258,13 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
|
||||
if am.agentChannelAlert != nil && !exclude[am.agentChannelAlert.ID] {
|
||||
result = append(result, *am.agentChannelAlert)
|
||||
}
|
||||
// Dead-app banners (fix-3) — prepended after the channel alert: a deployed app being down is a
|
||||
// high-signal state the operator/customer must see immediately.
|
||||
for _, a := range am.deadAppAlerts {
|
||||
if !exclude[a.ID] {
|
||||
result = append(result, a)
|
||||
}
|
||||
}
|
||||
for _, a := range am.alerts {
|
||||
if exclude[a.ID] {
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user