controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
This commit is contained in:
@@ -39,6 +39,16 @@ type Notifier struct {
|
||||
mu sync.Mutex
|
||||
prevHealthStatus string // tracks previous health check status for change detection
|
||||
|
||||
// appDown tracks which deployed apps are currently in the DOWN state so app_start_failed fires
|
||||
// ONCE per running→down transition, not every health cycle (fix-3 anti-spam). In-memory: a
|
||||
// controller restart re-notifies once (acceptable — better than missing). The hub owns the real
|
||||
// cooldown; the controller must not add its own timer.
|
||||
appDown map[string]bool
|
||||
|
||||
// pushFn is a test seam for the transition-emitting notifiers (fix-3). nil → the real async
|
||||
// PushEvent; tests inject a synchronous recorder.
|
||||
pushFn func(eventType, severity, message string, details interface{})
|
||||
|
||||
// Event history ring buffer (debug page)
|
||||
historyMu sync.RWMutex
|
||||
history [50]EventHistoryEntry
|
||||
@@ -362,6 +372,62 @@ func (n *Notifier) NotifyAppDeployed(stackName, displayName string) {
|
||||
AppDetails{StackName: stackName, DisplayName: displayName})
|
||||
}
|
||||
|
||||
// AppRunState is one deployed app's running state for the fix-3 start-failure notifier: Down=true
|
||||
// when the app is deployed but its containers are not running.
|
||||
type AppRunState struct {
|
||||
Name string
|
||||
DisplayName string
|
||||
Down bool
|
||||
}
|
||||
|
||||
// NotifyAppStartFailures fires an `app_start_failed` hub event ONCE per running→down transition
|
||||
// (fix-3). It is called each health cycle with the CURRENT deployed-app run states; the per-app
|
||||
// transition tracking (n.appDown) makes down→down cycles silent, so a persistently-dead app does not
|
||||
// spam. down→running clears the tracker (no event — the dashboard banner self-clears; a recovery
|
||||
// event is deliberately omitted to keep the operator inbox quiet). The hub applies its own cooldown.
|
||||
func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
|
||||
n.mu.Lock()
|
||||
if n.appDown == nil {
|
||||
n.appDown = map[string]bool{}
|
||||
}
|
||||
var newlyDown []AppRunState
|
||||
seen := map[string]bool{}
|
||||
for _, a := range apps {
|
||||
seen[a.Name] = true
|
||||
was := n.appDown[a.Name]
|
||||
if a.Down && !was {
|
||||
newlyDown = append(newlyDown, a) // running→down (or first-seen-down after the boot grace)
|
||||
}
|
||||
n.appDown[a.Name] = a.Down
|
||||
}
|
||||
// Forget apps no longer reported (removed/undeployed) so a later redeploy re-notifies cleanly.
|
||||
for name := range n.appDown {
|
||||
if !seen[name] {
|
||||
delete(n.appDown, name)
|
||||
}
|
||||
}
|
||||
n.mu.Unlock()
|
||||
|
||||
for _, a := range newlyDown {
|
||||
name := a.DisplayName
|
||||
if name == "" {
|
||||
name = a.Name
|
||||
}
|
||||
n.emit("app_start_failed", "warn",
|
||||
fmt.Sprintf("Telepített alkalmazás nem fut: %s", name),
|
||||
AppDetails{StackName: a.Name, DisplayName: a.DisplayName})
|
||||
}
|
||||
}
|
||||
|
||||
// emit sends an event through the test seam if set, else the real async PushEvent.
|
||||
func (n *Notifier) emit(eventType, severity, message string, details interface{}) {
|
||||
if n.pushFn != nil {
|
||||
n.pushFn(eventType, severity, message, details)
|
||||
return
|
||||
}
|
||||
n.PushEvent(eventType, severity, message, details)
|
||||
}
|
||||
|
||||
// NotifyAppRemoved sends an app removal event.
|
||||
func (n *Notifier) NotifyAppRemoved(stackName, displayName string) {
|
||||
n.PushEvent("app_removed", "info",
|
||||
|
||||
Reference in New Issue
Block a user