v0.6.0: healthcheck + hub reporting implementation
- Add heartbeat ping (every 5 min, controller alive signal) - Add backup integrity check (weekly restic check, Sunday 04:00) - Add Heartbeat + BackupIntegrity fields to PingUUIDsConfig - Add HubConfig for central hub reporting - Add report package (types, builder, pusher) for hub push - Wire hub reporting into scheduler (configurable interval) - Update controller.yaml.example with new monitoring + hub sections - Add monitoring/DEPRECATED.md for legacy bash scripts Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,230 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
||||
)
|
||||
|
||||
// BuildReport collects current state from all subsystems and returns a Report.
|
||||
func BuildReport(
|
||||
cfg *config.Config,
|
||||
stackMgr *stacks.Manager,
|
||||
backupMgr *backup.Manager,
|
||||
cpuCollector *system.CPUCollector,
|
||||
metricsStore *metrics.MetricsStore,
|
||||
version string,
|
||||
) *Report {
|
||||
r := &Report{
|
||||
Version: 1,
|
||||
CustomerID: cfg.Customer.ID,
|
||||
CustomerName: cfg.Customer.Name,
|
||||
ControllerVersion: version,
|
||||
Timestamp: time.Now().UTC(),
|
||||
}
|
||||
|
||||
// System info
|
||||
staticInfo := metrics.GetStaticInfo()
|
||||
sysInfo := system.GetInfo(cfg.Paths.HDDPath, cpuCollector)
|
||||
|
||||
r.System = SystemReport{
|
||||
Hostname: staticInfo.Hostname,
|
||||
OS: staticInfo.OS,
|
||||
Kernel: staticInfo.Kernel,
|
||||
CPUModel: staticInfo.CPUModel,
|
||||
CPUCores: staticInfo.CPUCores,
|
||||
UptimeSeconds: staticInfo.UptimeSeconds,
|
||||
CPUPercent: sysInfo.CPUPercent,
|
||||
MemoryTotalMB: sysInfo.TotalMemMB,
|
||||
MemoryUsedMB: sysInfo.UsedMemMB,
|
||||
MemoryPercent: sysInfo.MemPercent,
|
||||
TemperatureCelsius: sysInfo.TemperatureCelsius,
|
||||
LoadAvg1: sysInfo.LoadAvg1,
|
||||
LoadAvg5: sysInfo.LoadAvg5,
|
||||
LoadAvg15: sysInfo.LoadAvg15,
|
||||
}
|
||||
|
||||
// Storage
|
||||
r.Storage = []StorageReport{
|
||||
{Mount: "/", TotalGB: sysInfo.DiskTotalGB, UsedGB: sysInfo.DiskUsedGB, Percent: sysInfo.DiskPercent},
|
||||
}
|
||||
if sysInfo.HDDConfigured {
|
||||
r.Storage = append(r.Storage, StorageReport{
|
||||
Mount: cfg.Paths.HDDPath,
|
||||
TotalGB: sysInfo.HDDTotalGB,
|
||||
UsedGB: sysInfo.HDDUsedGB,
|
||||
Percent: sysInfo.HDDPercent,
|
||||
})
|
||||
}
|
||||
|
||||
// Containers
|
||||
r.Containers = buildContainerReport(stackMgr, metricsStore)
|
||||
|
||||
// Backup
|
||||
r.Backup = buildBackupReport(cfg, backupMgr)
|
||||
|
||||
// Health
|
||||
healthReport := monitor.RunHealthCheck(cfg, cpuCollector)
|
||||
r.Health = HealthReport{
|
||||
Status: healthReport.Status,
|
||||
Issues: healthReport.Issues,
|
||||
Warnings: healthReport.Warnings,
|
||||
}
|
||||
if r.Health.Issues == nil {
|
||||
r.Health.Issues = []string{}
|
||||
}
|
||||
if r.Health.Warnings == nil {
|
||||
r.Health.Warnings = []string{}
|
||||
}
|
||||
|
||||
// Stacks
|
||||
r.Stacks = buildStacksReport(stackMgr)
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func buildContainerReport(stackMgr *stacks.Manager, metricsStore *metrics.MetricsStore) ContainerReport {
|
||||
cr := ContainerReport{}
|
||||
|
||||
allStacks := stackMgr.GetStacks()
|
||||
|
||||
// Build a map of container stats from metrics store
|
||||
statsMap := make(map[string]metrics.ContainerCurrentStats)
|
||||
if metricsStore != nil {
|
||||
if stats, err := metricsStore.QueryContainerSummary(); err == nil {
|
||||
for _, s := range stats {
|
||||
statsMap[s.ContainerName] = s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range allStacks {
|
||||
if !s.Deployed {
|
||||
continue
|
||||
}
|
||||
for _, c := range s.Containers {
|
||||
cr.Total++
|
||||
switch c.State {
|
||||
case stacks.StateRunning, stacks.StateStarting:
|
||||
cr.Running++
|
||||
case stacks.StateUnhealthy:
|
||||
cr.Unhealthy++
|
||||
cr.Running++ // unhealthy containers are still running
|
||||
default:
|
||||
cr.Stopped++
|
||||
}
|
||||
|
||||
detail := ContainerDetailReport{
|
||||
Name: c.Name,
|
||||
State: string(c.State),
|
||||
}
|
||||
if cs, ok := statsMap[c.Name]; ok {
|
||||
detail.CPUPercent = cs.CPUPercent
|
||||
detail.MemoryMB = cs.MemUsageMB
|
||||
}
|
||||
cr.List = append(cr.List, detail)
|
||||
}
|
||||
}
|
||||
|
||||
if cr.List == nil {
|
||||
cr.List = []ContainerDetailReport{}
|
||||
}
|
||||
|
||||
return cr
|
||||
}
|
||||
|
||||
func buildBackupReport(cfg *config.Config, backupMgr *backup.Manager) BackupReport {
|
||||
br := BackupReport{
|
||||
Enabled: cfg.Backup.Enabled,
|
||||
}
|
||||
|
||||
if backupMgr == nil {
|
||||
return br
|
||||
}
|
||||
|
||||
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
|
||||
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
|
||||
status := backupMgr.GetFullStatus(nextDBDump, nextBackup)
|
||||
|
||||
if status.LastDBDump != nil {
|
||||
t := status.LastDBDump.LastRun
|
||||
br.LastDBDump = &t
|
||||
}
|
||||
if status.LastBackup != nil {
|
||||
t := status.LastBackup.LastRun
|
||||
br.LastSnapshot = &t
|
||||
}
|
||||
if status.RepoStats != nil {
|
||||
br.SnapshotCount = status.RepoStats.SnapshotCount
|
||||
br.RepoSizeMB = parseSizeToMB(status.RepoStats.TotalSize)
|
||||
}
|
||||
if !status.LastCheckTime.IsZero() {
|
||||
t := status.LastCheckTime
|
||||
br.LastIntegrityCheck = &t
|
||||
}
|
||||
br.IntegrityOK = status.LastCheckOK
|
||||
|
||||
return br
|
||||
}
|
||||
|
||||
func buildStacksReport(stackMgr *stacks.Manager) StacksReport {
|
||||
sr := StacksReport{}
|
||||
allStacks := stackMgr.GetStacks()
|
||||
|
||||
for _, s := range allStacks {
|
||||
if s.Protected {
|
||||
continue
|
||||
}
|
||||
if s.Deployed {
|
||||
sr.Deployed = append(sr.Deployed, s.Name)
|
||||
} else {
|
||||
sr.Available = append(sr.Available, s.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if sr.Deployed == nil {
|
||||
sr.Deployed = []string{}
|
||||
}
|
||||
if sr.Available == nil {
|
||||
sr.Available = []string{}
|
||||
}
|
||||
|
||||
return sr
|
||||
}
|
||||
|
||||
// parseSizeToMB parses a formatted size string like "1.5 GB", "512.0 MB" into MB.
|
||||
func parseSizeToMB(s string) int64 {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
parts := strings.Fields(s)
|
||||
if len(parts) != 2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
val, err := strconv.ParseFloat(parts[0], 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
switch strings.ToUpper(parts[1]) {
|
||||
case "GB":
|
||||
return int64(val * 1024)
|
||||
case "MB":
|
||||
return int64(val)
|
||||
case "KB":
|
||||
return int64(val / 1024)
|
||||
default:
|
||||
return int64(val)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user