channelhealth: controller->agent channel health-check (periodic probe + classified operator alert) v0.90.0
New internal/channelhealth Checker: ~60s probe via the PRODUCTION memoized client (Server.ProbeAgentChannel, GET /storage), classifies failures (spike Q1 map), debounces transient reasons (N>=2; construction error latches distinctly), seeds first obs, alerts operator+dashboard on transition. Notifier.NotifyAgentChannelDown/Recovered (English, operator-only), AlertManager dashboard banner (Hungarian). No agent/hub change. Spike-proven. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EPZ4GJ8L5Jqf8UiPwbn1kt
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
@@ -59,6 +60,22 @@ func (s *Server) agentClient() (*agentapi.Client, error) {
|
||||
return s.agentCli, s.agentCliErr
|
||||
}
|
||||
|
||||
// ProbeAgentChannel runs one controller→agent channel health probe using the PRODUCTION memoized
|
||||
// client (NOT a fresh one — spike SPIKE-controller-agent-channel-health-2026-06-29: it self-heals,
|
||||
// reflects exactly what the disk UI sees, and avoids the per-call transport leak the singleton fixed).
|
||||
// It returns whether the failure was a CONSTRUCTION error (agentClient() couldn't build — a latching
|
||||
// config fault, distinct from a runtime channel failure) and the error (nil = channel up). The probe
|
||||
// is GET /storage (cheap, read-only — the same call probeLocalAPI uses at startup). This is the
|
||||
// channelhealth.Probe seam.
|
||||
func (s *Server) ProbeAgentChannel(ctx context.Context) (constructionErr bool, err error) {
|
||||
client, cerr := s.agentClient()
|
||||
if cerr != nil {
|
||||
return true, cerr
|
||||
}
|
||||
_, serr := client.Storage(ctx)
|
||||
return false, serr
|
||||
}
|
||||
|
||||
// writeDiskJSON writes the standard {ok,data,error} envelope used by the disk API.
|
||||
func writeDiskJSON(w http.ResponseWriter, status int, ok bool, errMsg string, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
@@ -33,6 +33,10 @@ type AlertManager struct {
|
||||
alerts []Alert
|
||||
logger *log.Logger
|
||||
hubPushStatusFn func() HubPushStatusData
|
||||
// agentChannelAlert is set/cleared by the channel-health checker (out-of-band from Refresh, which
|
||||
// is health-report-driven). nil = channel up. It is prepended in GetAlerts so a dead controller→
|
||||
// agent link (disk/storage UI broken) is always visible regardless of the health-report cycle.
|
||||
agentChannelAlert *Alert
|
||||
}
|
||||
|
||||
// NewAlertManager creates a new AlertManager.
|
||||
@@ -49,6 +53,25 @@ func (am *AlertManager) SetHubPushStatus(fn func() HubPushStatusData) {
|
||||
am.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetAgentChannelAlert sets (down=true) or clears (down=false) the controller→agent channel-down
|
||||
// dashboard banner. Called by the channel-health checker each probe (idempotent). msg is the short
|
||||
// Hungarian display line.
|
||||
func (am *AlertManager) SetAgentChannelAlert(down bool, msg string) {
|
||||
am.mu.Lock()
|
||||
defer am.mu.Unlock()
|
||||
if !down {
|
||||
am.agentChannelAlert = nil
|
||||
return
|
||||
}
|
||||
am.agentChannelAlert = &Alert{
|
||||
ID: "agent-channel-down",
|
||||
Level: "error",
|
||||
Message: msg,
|
||||
Link: "/settings",
|
||||
LinkText: "Beállítások",
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh regenerates alerts from the latest health check report and config state.
|
||||
// Called after each health check cycle (every 5 minutes) and on storage state changes.
|
||||
func (am *AlertManager) Refresh(report *monitor.HealthReport, cfg *config.Config, backupMgr *backup.Manager, updateAvailable bool, latestVersion string, storagePaths ...[]settings.StoragePath) {
|
||||
@@ -159,7 +182,7 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
|
||||
am.mu.RLock()
|
||||
defer am.mu.RUnlock()
|
||||
|
||||
if len(am.alerts) == 0 {
|
||||
if len(am.alerts) == 0 && am.agentChannelAlert == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -169,6 +192,10 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
|
||||
}
|
||||
|
||||
var result []Alert
|
||||
// Channel-down is prepended (highest priority — the agent link being dead breaks disk/storage UI).
|
||||
if am.agentChannelAlert != nil && !exclude[am.agentChannelAlert.ID] {
|
||||
result = append(result, *am.agentChannelAlert)
|
||||
}
|
||||
for _, a := range am.alerts {
|
||||
if exclude[a.ID] {
|
||||
continue
|
||||
@@ -239,4 +266,3 @@ func countLevel(alerts []Alert, level string) int {
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user