Files
felhom-controller/controller/internal/notify/notifier.go
T
admin 86ea482fc1 controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a
licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY
agent, and refuses it for AgeStateUnknown. An unreadable storage no longer
masquerades as a first-ever backup and no longer quiesces apps outside the window.

A missing wire field means legacy, not unknown — deliberately. Treating it as
unknown would stop the valve firing on un-upgraded boxes and starve genuinely new
ones. Degrade logged once; unrecognised future values also map to legacy.

Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the
signature change compiled and vetted clean while quiesceBackend silently stopped
satisfying it — which would have degraded every box to the single-tier path with
no error. Added a compile-time witness.

Also corrects the notifier comment that claimed operator-only came from a missing
customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
2026-07-27 18:08:56 +02:00

811 lines
28 KiB
Go

package notify
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// Notifier sends structured events to the hub via /api/v1/event.
// Non-blocking: fires requests in goroutines, logs errors but doesn't retry aggressively.
// Cooldown logic is handled by the Hub — the controller sends all events unconditionally.
// EventHistoryEntry records a sent event for the debug page.
type EventHistoryEntry struct {
Timestamp time.Time `json:"timestamp"`
EventType string `json:"event_type"`
Severity string `json:"severity"`
Message string `json:"message"`
HubStatus int `json:"hub_status"`
HubError string `json:"hub_error,omitempty"`
}
type Notifier struct {
hubURL string
apiKey string
customerID string
httpClient *http.Client
logger *log.Logger
enabled bool
debug bool
settings *settings.Settings
mu sync.Mutex
prevHealthStatus string // tracks previous health check status for change detection
// appDown tracks which deployed apps are currently in the DOWN state so app_start_failed fires
// ONCE per running→down transition, not every health cycle (fix-3 anti-spam). In-memory: a
// controller restart re-notifies once (acceptable — better than missing). The hub owns the real
// cooldown; the controller must not add its own timer.
appDown map[string]bool
// pushFn is a test seam for the transition-emitting notifiers (fix-3). nil → the real async
// PushEvent; tests inject a synchronous recorder.
pushFn func(eventType, severity, message string, details interface{})
// Event history ring buffer (debug page)
historyMu sync.RWMutex
history [50]EventHistoryEntry
histPos int
histFull bool
}
// New creates a new Notifier. Returns a no-op notifier if hub is not enabled.
func New(hubURL, apiKey, customerID string, sett *settings.Settings, logger *log.Logger, debug bool) *Notifier {
enabled := hubURL != "" && apiKey != ""
if enabled {
logger.Printf("[INFO] Notifier enabled (hub: %s)", hubURL)
} else {
logger.Printf("[INFO] Notifier disabled (hub not configured)")
}
return &Notifier{
hubURL: hubURL,
apiKey: apiKey,
customerID: customerID,
httpClient: &http.Client{Timeout: 10 * time.Second},
logger: logger,
enabled: enabled,
debug: debug,
settings: sett,
}
}
// IsEnabled returns whether the notifier has a configured hub connection.
func (n *Notifier) IsEnabled() bool {
return n.enabled
}
// ── Detail structs ───────────────────────────────────────────────────
// BackupDetails holds structured data for backup events.
type BackupDetails struct {
DriveCount int `json:"drive_count,omitempty"`
SnapshotID string `json:"snapshot_id,omitempty"`
DurationSec int `json:"duration_sec,omitempty"`
DataAdded string `json:"data_added,omitempty"`
Error string `json:"error,omitempty"`
}
// WholeGuestBackupDetails carries the TIER for a whole-guest (vzdump) backup event (R-97a).
//
// The `tier` field is load-bearing beyond display: the hub's operator cooldown is keyed
// `customerID:eventType` plus this tier when present, so `local` failing does not get swallowed by
// `felhom-pbs` having failed within the same hour. Rename it and the two tiers silently share one
// cooldown again.
type WholeGuestBackupDetails struct {
Tier string `json:"tier"`
Error string `json:"error,omitempty"`
}
// DBDumpDetails holds structured data for DB dump events.
type DBDumpDetails struct {
DatabaseCount int `json:"database_count,omitempty"`
TotalSize string `json:"total_size,omitempty"`
DurationSec int `json:"duration_sec,omitempty"`
Error string `json:"error,omitempty"`
}
// DiskDetails holds structured data for disk warning/critical events.
type DiskDetails struct {
Mount string `json:"mount,omitempty"`
UsagePercent float64 `json:"usage_percent,omitempty"`
Label string `json:"label,omitempty"`
}
// HealthDetails holds structured data for health events.
type HealthDetails struct {
PreviousStatus string `json:"previous_status,omitempty"`
CurrentStatus string `json:"current_status,omitempty"`
Issues []string `json:"issues,omitempty"`
Warnings []string `json:"warnings,omitempty"`
}
// StorageDetails holds structured data for storage events.
type StorageDetails struct {
DrivePath string `json:"drive_path,omitempty"`
Label string `json:"label,omitempty"`
StoppedApps []string `json:"stopped_apps,omitempty"`
}
// UpdateDetails holds structured data for controller update events.
type UpdateDetails struct {
FromVersion string `json:"from_version,omitempty"`
ToVersion string `json:"to_version,omitempty"`
Error string `json:"error,omitempty"`
}
// AppDetails holds structured data for app lifecycle events.
type AppDetails struct {
StackName string `json:"stack_name,omitempty"`
DisplayName string `json:"display_name,omitempty"`
}
// CrossDriveDetails holds structured data for cross-drive backup events.
type CrossDriveDetails struct {
StackName string `json:"stack_name,omitempty"`
Method string `json:"method,omitempty"`
DestPath string `json:"dest_path,omitempty"`
Duration string `json:"duration,omitempty"`
Error string `json:"error,omitempty"`
}
// ── Core event push ──────────────────────────────────────────────────
// eventRequest is the JSON payload sent to /api/v1/event.
type eventRequest struct {
CustomerID string `json:"customer_id"`
EventType string `json:"event_type"`
Severity string `json:"severity"`
Message string `json:"message"`
Details json.RawMessage `json:"details,omitempty"`
}
// PushEvent sends a structured event to the hub's /api/v1/event endpoint.
// Non-blocking (goroutine). Retries twice with 3s backoff.
// details may be nil (omitted from JSON) or a struct that marshals to JSON.
func (n *Notifier) PushEvent(eventType, severity, message string, details interface{}) {
if !n.enabled {
return
}
var detailsJSON json.RawMessage
if details != nil {
b, err := json.Marshal(details)
if err != nil {
n.logger.Printf("[WARN] PushEvent: failed to marshal details for %s: %v", eventType, err)
} else {
detailsJSON = b
}
}
payload := eventRequest{
CustomerID: n.customerID,
EventType: eventType,
Severity: severity,
Message: message,
Details: detailsJSON,
}
jsonData, err := json.Marshal(payload)
if err != nil {
n.logger.Printf("[ERROR] PushEvent: marshal failed for %s: %v", eventType, err)
return
}
go func() {
url := n.hubURL + "/api/v1/event"
if n.debug {
n.logger.Printf("[DEBUG] PushEvent: type=%s severity=%s url=%s", eventType, severity, url)
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
time.Sleep(3 * time.Second)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(jsonData))
if err != nil {
lastErr = err
continue
}
req.Header.Set("Authorization", "Bearer "+n.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := n.httpClient.Do(req)
if err != nil {
lastErr = err
continue
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
if n.debug {
n.logger.Printf("[DEBUG] PushEvent: %s pushed OK (HTTP %d)", eventType, resp.StatusCode)
}
n.logger.Printf("[INFO] Event pushed: %s (%s) — %s", eventType, severity, message)
n.recordHistory(eventType, severity, message, resp.StatusCode, "")
return
}
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
}
n.logger.Printf("[WARN] Event push failed after 3 attempts (%s/%s): %v", eventType, severity, lastErr)
errMsg := ""
if lastErr != nil {
errMsg = lastErr.Error()
}
n.recordHistory(eventType, severity, message, 0, errMsg)
}()
}
// ── Convenience methods ──────────────────────────────────────────────
// NotifyHealthChange checks if health status changed and sends appropriate events.
// Detects both degradation (ok→warn, ok→fail, warn→fail) and recovery (fail→ok, warn→ok, fail→warn).
func (n *Notifier) NotifyHealthChange(status string, issues, warnings []string) {
if !n.enabled {
return
}
n.mu.Lock()
prev := n.prevHealthStatus
n.prevHealthStatus = status
n.mu.Unlock()
if prev == "" {
return // First run, just record status
}
if status == prev {
return
}
details := HealthDetails{
PreviousStatus: prev,
CurrentStatus: status,
Issues: issues,
Warnings: warnings,
}
prevRank := statusRank(prev)
newRank := statusRank(status)
if newRank > prevRank {
// Degradation
if status == "fail" {
n.PushEvent("health_critical", "error",
fmt.Sprintf("Rendszer állapot kritikus (volt: %s)", prev), details)
} else if status == "warn" {
n.PushEvent("health_degraded", "warning",
fmt.Sprintf("Rendszer állapot romlott (volt: %s)", prev), details)
}
} else {
// Recovery
n.PushEvent("health_recovered", "info",
fmt.Sprintf("Rendszer állapot helyreállt: %s (volt: %s)", status, prev), details)
}
}
// NotifyBackupFailed sends a backup failure event.
func (n *Notifier) NotifyBackupFailed(message, errMsg string) {
n.PushEvent("backup_failed", "error", message, BackupDetails{Error: errMsg})
}
// NotifyOffboxEnlargeBlocked sends a WARNING (not a failure) when an app's enlarged offsite push was
// refused by the pre-push quota gate — its config+DB were still saved. Customer-facing (Hungarian
// body). NOTE: the event type "offbox_enlarge_blocked" must be added to the hub's allowedEventTypes +
// customerMessages for delivery (a hub-side task, flagged — until then the hub 400s/drops it and the
// in-dashboard LastWarning + /backups/remote note carry the message).
func (n *Notifier) NotifyOffboxEnlargeBlocked(message string) {
n.PushEvent("offbox_enlarge_blocked", "warning", message, nil)
}
// (NotifyBackupCompleted removed 2026-06-16 — the backup_completed event had no callers
// since slice 8C moved whole-guest backup to the agent. The hub's backup-deadline check
// now reads the agent host-report's PBS snapshots instead of this event. DB-dump events
// below are still emitted and consumed.)
// NotifyDBDumpFailed sends a DB dump failure event.
func (n *Notifier) NotifyDBDumpFailed(message, errMsg string) {
n.PushEvent("db_dump_failed", "error", message, DBDumpDetails{Error: errMsg})
}
// NotifyDBDumpCompleted sends a DB dump success event.
func (n *Notifier) NotifyDBDumpCompleted(details DBDumpDetails) {
n.PushEvent("db_dump_completed", "info", "Adatbázis mentés elkészült", details)
}
// NotifyIntegrityFailed sends a backup integrity check failure event.
func (n *Notifier) NotifyIntegrityFailed(message, errMsg string) {
n.PushEvent("backup_integrity_failed", "error", message, &BackupDetails{Error: errMsg})
}
// NotifyIntegrityOK sends a backup integrity check success event.
func (n *Notifier) NotifyIntegrityOK(message string) {
n.PushEvent("backup_integrity_ok", "info", message, nil)
}
// NotifyControllerUpdated sends a controller update event.
func (n *Notifier) NotifyControllerUpdated(fromVer, toVer string, success bool) {
severity := "info"
msg := fmt.Sprintf("Controller frissítve: %s → %s", fromVer, toVer)
details := UpdateDetails{FromVersion: fromVer, ToVersion: toVer}
if !success {
severity = "error"
msg = fmt.Sprintf("Controller frissítés sikertelen: %s → %s", fromVer, toVer)
}
n.PushEvent("controller_updated", severity, msg, details)
}
// NotifyControllerStarted sends a controller startup event.
// details may include self-test summary (e.g., {"selftest_pass": 8, "selftest_warn": 1, "selftest_fail": 0}).
func (n *Notifier) NotifyControllerStarted(version string, details map[string]interface{}) {
n.PushEvent("controller_started", "info",
fmt.Sprintf("Controller elindult (%s)", version), details)
}
// NotifyStorageDisconnected sends a drive disconnection event.
func (n *Notifier) NotifyStorageDisconnected(label string, stoppedApps []string) {
msg := fmt.Sprintf("Meghajtó váratlanul leválasztva: %s", label)
n.PushEvent("storage_disconnected", "error", msg, StorageDetails{
Label: label,
StoppedApps: stoppedApps,
})
}
// NotifyStorageReconnected sends a drive reconnection event.
func (n *Notifier) NotifyStorageReconnected(label string) {
n.PushEvent("storage_reconnected", "info",
fmt.Sprintf("Meghajtó újra csatlakoztatva: %s", label), StorageDetails{Label: label})
}
// AgentChannelDetails carries the classified reason for a controller→agent channel-down event.
type AgentChannelDetails struct {
Reason string `json:"reason"`
}
// NotifyAgentChannelDown sends an OPERATOR-facing controller→agent channel-down event (the message is
// English — the customer can't act on "the agent re-keyed", so this never reaches them: the event type
// is not a customer notification toggle, same as the host_* operator events). eventType/severity/
// message come from the channelhealth classifier (spike Q1 map).
func (n *Notifier) NotifyAgentChannelDown(reason, eventType, severity, message string) {
n.PushEvent(eventType, severity, message, AgentChannelDetails{Reason: reason})
}
// NotifyAgentChannelRecovered sends the recovery event (info → logged, no email, mirrors
// storage_reconnected).
func (n *Notifier) NotifyAgentChannelRecovered() {
n.PushEvent("agent_channel_recovered", "info",
"Controller→agent channel recovered — local-API reachable again.", nil)
}
// NotifyEndpointDrift reports a local_api endpoint divergence (R-77). Operator-only English, its
// OWN event type — deliberately not folded into agent_channel_*, because during the 2026-07-25
// outage the generic channel alert was the only signal and it hid a specific, fixable config fault.
// severity=error: unlike a transient unreachable, drift never self-heals.
//
// NOTE: the hub validates event_type against allowedEventTypes and 400s an unknown one, so this
// type MUST exist there too (hub handler.go) or the alert is silently inert.
func (n *Notifier) NotifyEndpointDrift(message string, fingerprintAgrees bool) {
n.PushEvent("local_api_endpoint_drift", "error", message,
EndpointDriftDetails{FingerprintAgrees: fingerprintAgrees})
}
// EndpointDriftDetails carries NO addresses and NO secrets — the endpoints are in the message, and
// the pin is a boolean by design.
type EndpointDriftDetails struct {
FingerprintAgrees bool `json:"fingerprint_agrees"`
}
// NotifyAppDeployed sends an app deployment event.
func (n *Notifier) NotifyAppDeployed(stackName, displayName string) {
n.PushEvent("app_deployed", "info",
fmt.Sprintf("Alkalmazás telepítve: %s", displayName),
AppDetails{StackName: stackName, DisplayName: displayName})
}
// AppRunState is one deployed app's running state for the fix-3 start-failure notifier: Down=true
// when the app is deployed but its containers are not running.
type AppRunState struct {
Name string
DisplayName string
Down bool
}
// NotifyAppStartFailures fires an `app_start_failed` hub event ONCE per running→down transition
// (fix-3). It is called each health cycle with the CURRENT deployed-app run states; the per-app
// transition tracking (n.appDown) makes down→down cycles silent, so a persistently-dead app does not
// spam. down→running clears the tracker (no event — the dashboard banner self-clears; a recovery
// event is deliberately omitted to keep the operator inbox quiet). The hub applies its own cooldown.
func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
n.mu.Lock()
if n.appDown == nil {
n.appDown = map[string]bool{}
}
var newlyDown []AppRunState
seen := map[string]bool{}
for _, a := range apps {
seen[a.Name] = true
was := n.appDown[a.Name]
if a.Down && !was {
newlyDown = append(newlyDown, a) // running→down (or first-seen-down after the boot grace)
}
n.appDown[a.Name] = a.Down
}
// Forget apps no longer reported (removed/undeployed) so a later redeploy re-notifies cleanly.
for name := range n.appDown {
if !seen[name] {
delete(n.appDown, name)
}
}
n.mu.Unlock()
for _, a := range newlyDown {
name := a.DisplayName
if name == "" {
name = a.Name
}
n.emit("app_start_failed", "warn",
fmt.Sprintf("Telepített alkalmazás nem fut: %s", name),
AppDetails{StackName: a.Name, DisplayName: a.DisplayName})
}
}
// DiskHealthDetails is the event-detail payload for disk_health_degraded.
type DiskHealthDetails struct {
Disk string `json:"disk"`
Attributes []string `json:"attributes,omitempty"`
Critical bool `json:"critical"`
}
// NotifyDiskHealthDegraded fires a disk_health_degraded hub event on a disk-health DEGRADATION only
// (the controller's 6h check owns the transition logic — never first-run/recovery/UNKNOWN). severity
// is warn for a Figyelmeztetés, critical for a Hiba (FAILING). The hub applies its own per-event-type
// cooldown. NOTE: the event type "disk_health_degraded" MUST be in the hub's allowedEventTypes (else
// the hub 400s the POST) — added in the hub's v0.x bump alongside this.
func (n *Notifier) NotifyDiskHealthDegraded(label string, attrs []string, critical bool) {
severity := "warn"
var msg string
switch {
case critical:
severity = "critical"
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez SMART önellenőrzése hibát jelez. Kérjük, mentse az adatait, és vegye fel velünk a kapcsolatot.", label)
case len(attrs) > 0:
msg = fmt.Sprintf("Lemez állapot romlás: %s — romló érték: %s. Javasolt figyelemmel kísérni.", label, strings.Join(attrs, ", "))
default:
msg = fmt.Sprintf("Lemez állapot romlás: %s — a lemez állapota romlott. Javasolt figyelemmel kísérni.", label)
}
n.emit("disk_health_degraded", severity, msg, DiskHealthDetails{Disk: label, Attributes: attrs, Critical: critical})
}
// emit sends an event through the test seam if set, else the real async PushEvent.
func (n *Notifier) emit(eventType, severity, message string, details interface{}) {
if n.pushFn != nil {
n.pushFn(eventType, severity, message, details)
return
}
n.PushEvent(eventType, severity, message, details)
}
// NotifyAppRemoved sends an app removal event.
func (n *Notifier) NotifyAppRemoved(stackName, displayName string) {
n.PushEvent("app_removed", "info",
fmt.Sprintf("Alkalmazás eltávolítva: %s", displayName),
AppDetails{StackName: stackName, DisplayName: displayName})
}
// NotifyCrossDriveCompleted sends a cross-drive backup success event.
func (n *Notifier) NotifyCrossDriveCompleted(details CrossDriveDetails) {
n.PushEvent("crossdrive_completed", "info",
fmt.Sprintf("Másodlagos mentés elkészült: %s", details.StackName), details)
}
// NotifyCrossDriveFailed sends a cross-drive backup failure event.
func (n *Notifier) NotifyCrossDriveFailed(details CrossDriveDetails) {
n.PushEvent("crossdrive_failed", "error",
fmt.Sprintf("Másodlagos mentés sikertelen: %s", details.StackName), details)
}
// NotifyDRStarted sends a disaster recovery start event.
func (n *Notifier) NotifyDRStarted(appCount int) {
n.PushEvent("disaster_recovery_started", "warning",
fmt.Sprintf("Katasztrófa helyreállítás elindítva (%d alkalmazás)", appCount), nil)
}
// NotifyDRCompleted sends a disaster recovery completion event.
func (n *Notifier) NotifyDRCompleted(successCount, failCount int) {
severity := "info"
if failCount > 0 {
severity = "warning"
}
n.PushEvent("disaster_recovery_completed", severity,
fmt.Sprintf("Katasztrófa helyreállítás befejezve (%d sikeres, %d sikertelen)", successCount, failCount), nil)
}
// ── Preferences sync ─────────────────────────────────────────────────
type preferencesRequest struct {
CustomerID string `json:"customer_id"`
Email string `json:"email"`
EnabledEvents []string `json:"enabled_events"`
CooldownHours int `json:"cooldown_hours,omitempty"`
}
// SyncPreferences pushes the current notification preferences to the hub.
// Synchronous — returns error for the handler to display to the user.
func (n *Notifier) SyncPreferences(email string, enabledEvents []string, cooldownHours int) error {
if !n.enabled {
return fmt.Errorf("hub nem konfigurált")
}
payload := preferencesRequest{
CustomerID: n.customerID,
Email: email,
EnabledEvents: enabledEvents,
CooldownHours: cooldownHours,
}
jsonData, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal: %w", err)
}
url := n.hubURL + "/api/v1/preferences"
if n.debug {
n.logger.Printf("[DEBUG] SyncPreferences: url=%s email=%s events=%v cooldown=%dh",
url, email, enabledEvents, cooldownHours)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(jsonData))
if err != nil {
return fmt.Errorf("request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+n.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := n.httpClient.Do(req)
if err != nil {
return fmt.Errorf("hub elérhetetlen: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("hub hiba (%d): %s", resp.StatusCode, string(body))
}
if n.debug {
n.logger.Printf("[DEBUG] SyncPreferences: response HTTP %d", resp.StatusCode)
}
n.logger.Printf("[INFO] Notification preferences synced to hub: email=%s, events=%v, cooldown=%dh", email, enabledEvents, cooldownHours)
return nil
}
// ── Test notification ────────────────────────────────────────────────
// SendTest sends a test event for verifying the notification flow (synchronous).
func (n *Notifier) SendTest() error {
if !n.enabled {
return fmt.Errorf("notifications not enabled (hub not configured)")
}
payload := eventRequest{
CustomerID: n.customerID,
EventType: "test",
Severity: "info",
Message: "Teszt értesítés a Felhom rendszerből",
}
jsonData, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal: %w", err)
}
url := n.hubURL + "/api/v1/event"
req, err := http.NewRequest("POST", url, bytes.NewReader(jsonData))
if err != nil {
return fmt.Errorf("request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+n.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := n.httpClient.Do(req)
if err != nil {
return fmt.Errorf("send: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("hub returned %d", resp.StatusCode)
}
return nil
}
// ── Debug event testing ───────────────────────────────────────────────
// PushTestEventSync sends a test event synchronously and returns the Hub HTTP status code.
// Used by the debug page for event testing with configurable type/severity.
func (n *Notifier) PushTestEventSync(eventType, severity, message string) (statusCode int, err error) {
if !n.enabled {
return 0, fmt.Errorf("hub nem konfigurált")
}
payload := eventRequest{
CustomerID: n.customerID,
EventType: eventType,
Severity: severity,
Message: message,
}
jsonData, err := json.Marshal(payload)
if err != nil {
return 0, fmt.Errorf("marshal: %w", err)
}
url := n.hubURL + "/api/v1/event"
req, err := http.NewRequest("POST", url, bytes.NewReader(jsonData))
if err != nil {
return 0, fmt.Errorf("request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+n.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := n.httpClient.Do(req)
if err != nil {
n.recordHistory(eventType, severity, message, 0, err.Error())
return 0, fmt.Errorf("send: %w", err)
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 400 {
n.recordHistory(eventType, severity, message, resp.StatusCode, fmt.Sprintf("HTTP %d", resp.StatusCode))
return resp.StatusCode, fmt.Errorf("hub returned %d", resp.StatusCode)
}
n.recordHistory(eventType, severity, message, resp.StatusCode, "")
return resp.StatusCode, nil
}
// GetEventHistory returns the last N event history entries (newest first).
func (n *Notifier) GetEventHistory(limit int) []EventHistoryEntry {
n.historyMu.RLock()
defer n.historyMu.RUnlock()
total := n.histPos
if n.histFull {
total = len(n.history)
}
if limit <= 0 || limit > total {
limit = total
}
result := make([]EventHistoryEntry, 0, limit)
for i := 0; i < limit; i++ {
idx := n.histPos - 1 - i
if idx < 0 {
idx += len(n.history)
}
result = append(result, n.history[idx])
}
return result
}
// recordHistory appends an entry to the event history ring buffer.
func (n *Notifier) recordHistory(eventType, severity, message string, hubStatus int, hubError string) {
n.historyMu.Lock()
defer n.historyMu.Unlock()
n.history[n.histPos] = EventHistoryEntry{
Timestamp: time.Now(),
EventType: eventType,
Severity: severity,
Message: message,
HubStatus: hubStatus,
HubError: hubError,
}
n.histPos++
if n.histPos >= len(n.history) {
n.histPos = 0
n.histFull = true
}
}
// ── Backward compatibility ───────────────────────────────────────────
// notifyRequest is the JSON payload for the legacy /api/v1/notify endpoint.
type notifyRequest struct {
CustomerID string `json:"customer_id"`
EventType string `json:"event_type"`
Severity string `json:"severity"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
}
// Notify sends a legacy notification to /api/v1/notify (backward compat).
// Kept for old Hub instances that don't support /api/v1/event yet.
// No local cooldown — Hub handles cooldowns.
func (n *Notifier) Notify(eventType, severity, message, details string) {
if !n.enabled {
return
}
go func() {
payload := notifyRequest{
CustomerID: n.customerID,
EventType: eventType,
Severity: severity,
Message: message,
Details: details,
}
jsonData, err := json.Marshal(payload)
if err != nil {
n.logger.Printf("[ERROR] Failed to marshal notification: %v", err)
return
}
url := n.hubURL + "/api/v1/notify"
req, err := http.NewRequest("POST", url, bytes.NewReader(jsonData))
if err != nil {
return
}
req.Header.Set("Authorization", "Bearer "+n.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := n.httpClient.Do(req)
if err != nil {
return
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
}
// ── Helpers ──────────────────────────────────────────────────────────
func statusRank(status string) int {
switch status {
case "ok":
return 0
case "warn":
return 1
case "fail":
return 2
default:
return 0
}
}
// NotifyWholeGuestBackupFailed / ...Recovered — R-97a, the WHOLE-GUEST (vzdump) backup tier.
//
// OPERATOR-TIER ONLY, and that is why these are NOT `backup_failed`. `backup_failed` and
// `backup_completed` both carry `customerMessages` entries in the hub AND sit in demo-felhom's live
// `enabled_events`, so reusing them would email the CUSTOMER, in Hungarian, that their backup failed
// — while it is still retrying behind the R-88 breaker. A customer can take no action on a failed
// whole-guest backup; that is the same harm R-97b removes, re-introduced through the front door.
//
// Operator-only is enforced hub-side by `notify.operatorOnlyEvents` (hub >= v0.79.0, R-97c), NOT by
// the absence of a `customerMessages` entry — v0.177.0 claimed the latter and was WRONG: the hub
// falls back to the raw message when the entry is missing, and the only customer gate is
// `prefs.EnabledEvents`, which is configuration. Adding a type to the allowlist does NOT make it
// operator-only; it must go in that register too.
//
// HUB DEPENDENCY: both types MUST be present in the hub's allowedEventTypes or POST /event 400s
// (the recorded allowlist gotcha). Do not deploy this controller ahead of that hub change.
func (n *Notifier) NotifyWholeGuestBackupFailed(tier, message, errMsg string) {
n.PushEvent("whole_guest_backup_failed", "error", message,
WholeGuestBackupDetails{Tier: tier, Error: errMsg})
}
func (n *Notifier) NotifyWholeGuestBackupRecovered(tier, message string) {
n.PushEvent("whole_guest_backup_recovered", "info", message,
WholeGuestBackupDetails{Tier: tier})
}