controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
This commit is contained in:
2026-07-12 10:07:47 +02:00
parent a6da64da15
commit d8f6069b46
10 changed files with 581 additions and 10 deletions
+66
View File
@@ -39,6 +39,16 @@ type Notifier struct {
mu sync.Mutex
prevHealthStatus string // tracks previous health check status for change detection
// appDown tracks which deployed apps are currently in the DOWN state so app_start_failed fires
// ONCE per running→down transition, not every health cycle (fix-3 anti-spam). In-memory: a
// controller restart re-notifies once (acceptable — better than missing). The hub owns the real
// cooldown; the controller must not add its own timer.
appDown map[string]bool
// pushFn is a test seam for the transition-emitting notifiers (fix-3). nil → the real async
// PushEvent; tests inject a synchronous recorder.
pushFn func(eventType, severity, message string, details interface{})
// Event history ring buffer (debug page)
historyMu sync.RWMutex
history [50]EventHistoryEntry
@@ -362,6 +372,62 @@ func (n *Notifier) NotifyAppDeployed(stackName, displayName string) {
AppDetails{StackName: stackName, DisplayName: displayName})
}
// AppRunState is one deployed app's running state for the fix-3 start-failure notifier: Down=true
// when the app is deployed but its containers are not running.
type AppRunState struct {
Name string
DisplayName string
Down bool
}
// NotifyAppStartFailures fires an `app_start_failed` hub event ONCE per running→down transition
// (fix-3). It is called each health cycle with the CURRENT deployed-app run states; the per-app
// transition tracking (n.appDown) makes down→down cycles silent, so a persistently-dead app does not
// spam. down→running clears the tracker (no event — the dashboard banner self-clears; a recovery
// event is deliberately omitted to keep the operator inbox quiet). The hub applies its own cooldown.
func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
n.mu.Lock()
if n.appDown == nil {
n.appDown = map[string]bool{}
}
var newlyDown []AppRunState
seen := map[string]bool{}
for _, a := range apps {
seen[a.Name] = true
was := n.appDown[a.Name]
if a.Down && !was {
newlyDown = append(newlyDown, a) // running→down (or first-seen-down after the boot grace)
}
n.appDown[a.Name] = a.Down
}
// Forget apps no longer reported (removed/undeployed) so a later redeploy re-notifies cleanly.
for name := range n.appDown {
if !seen[name] {
delete(n.appDown, name)
}
}
n.mu.Unlock()
for _, a := range newlyDown {
name := a.DisplayName
if name == "" {
name = a.Name
}
n.emit("app_start_failed", "warn",
fmt.Sprintf("Telepített alkalmazás nem fut: %s", name),
AppDetails{StackName: a.Name, DisplayName: a.DisplayName})
}
}
// emit sends an event through the test seam if set, else the real async PushEvent.
func (n *Notifier) emit(eventType, severity, message string, details interface{}) {
if n.pushFn != nil {
n.pushFn(eventType, severity, message, details)
return
}
n.PushEvent(eventType, severity, message, details)
}
// NotifyAppRemoved sends an app removal event.
func (n *Notifier) NotifyAppRemoved(stackName, displayName string) {
n.PushEvent("app_removed", "info",