channelhealth: controller->agent channel health-check (periodic probe + classified operator alert) v0.90.0
New internal/channelhealth Checker: ~60s probe via the PRODUCTION memoized client (Server.ProbeAgentChannel, GET /storage), classifies failures (spike Q1 map), debounces transient reasons (N>=2; construction error latches distinctly), seeds first obs, alerts operator+dashboard on transition. Notifier.NotifyAgentChannelDown/Recovered (English, operator-only), AlertManager dashboard banner (Hungarian). No agent/hub change. Spike-proven. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt
This commit is contained in:
@@ -25,6 +25,7 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/assets"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
|
||||
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
|
||||
@@ -253,9 +254,9 @@ func main() {
|
||||
TLSAddr: cfg.MailRelay.TLSListen,
|
||||
PlainNoTLSAddr: cfg.MailRelay.PlainNoTLSListen,
|
||||
ServiceName: cfg.MailRelay.ShimHost,
|
||||
Policy: mailrelay.NewPolicy(cfg.MailRelay.FromDomains),
|
||||
Forwarder: mailrelay.NewHubForwarder(cfg.Hub.URL, cfg.Hub.APIKey),
|
||||
Logger: logger,
|
||||
Policy: mailrelay.NewPolicy(cfg.MailRelay.FromDomains),
|
||||
Forwarder: mailrelay.NewHubForwarder(cfg.Hub.URL, cfg.Hub.APIKey),
|
||||
Logger: logger,
|
||||
})
|
||||
if err := mailShim.Apply(sett.AppEmailEnabled()); err != nil {
|
||||
logger.Printf("[ERROR] [mailrelay] could not start app-email shim: %v", err)
|
||||
@@ -717,6 +718,17 @@ func main() {
|
||||
}
|
||||
webServer.SetStartTime(startTime)
|
||||
|
||||
// Controller→agent channel health (self-health slice). A ~60s probe of the local-API channel via
|
||||
// the PRODUCTION memoized client (webServer.ProbeAgentChannel), classified + debounced, alerting the
|
||||
// operator + dashboard on a transition. Only on a provisioned guest (endpoint set) — mirrors
|
||||
// probeLocalAPI's guard. Registered after webServer (it owns the memoized client); sched.Every
|
||||
// launches the job immediately since the scheduler is already started.
|
||||
if cfg.LocalAPI.Endpoint != "" {
|
||||
chSink := channelSink{notifier: notifier, alertMgr: alertMgr}
|
||||
chChecker := channelhealth.New(webServer.ProbeAgentChannel, chSink, logger)
|
||||
sched.Every("agent-channel-health", 60*time.Second, chChecker.Check)
|
||||
}
|
||||
|
||||
// Wire debug callbacks (only in debug mode)
|
||||
if cfg.Logging.Level == "debug" {
|
||||
dc := &web.DebugCallbacks{}
|
||||
@@ -1308,6 +1320,22 @@ func parseDurationOr(s string, def time.Duration) time.Duration {
|
||||
return d
|
||||
}
|
||||
|
||||
// channelSink adapts the notify.Notifier + web.AlertManager to the channelhealth.Sink seam (kept in
|
||||
// main.go so the channelhealth package imports neither — no cycle). SetDashboard reflects the current
|
||||
// state each probe (idempotent); NotifyDown/NotifyRecovered fire only on a transition.
|
||||
type channelSink struct {
|
||||
notifier *notify.Notifier
|
||||
alertMgr *web.AlertManager
|
||||
}
|
||||
|
||||
func (s channelSink) SetDashboard(down bool, _ channelhealth.Reason, msg string) {
|
||||
s.alertMgr.SetAgentChannelAlert(down, msg)
|
||||
}
|
||||
func (s channelSink) NotifyDown(reason channelhealth.Reason, eventType, severity, msg string) {
|
||||
s.notifier.NotifyAgentChannelDown(string(reason), eventType, severity, msg)
|
||||
}
|
||||
func (s channelSink) NotifyRecovered() { s.notifier.NotifyAgentChannelRecovered() }
|
||||
|
||||
// probeLocalAPI proves the controller↔agent local-API channel at startup and logs this guest's
|
||||
// mounts (slice 8A). Non-fatal: it only runs when a local-API endpoint is configured, and any
|
||||
// error is logged for diagnosis without affecting the controller's boot. The leaf SHA-256 from
|
||||
|
||||
Reference in New Issue
Block a user