bdbe170a54
New storage watchdog monitors registered storage paths every 5s. On disconnect (3 consecutive probe failures), auto-stops affected apps, lazy-unmounts stale VFS entries, fires alerts/notifications/hub report. On reconnect (UUID detected), auto-remounts via fstab, cleans stale restic locks, offers app restart. Safe disconnect UI for USB drives: confirmation dialog, stop apps, sync, unmount. Disconnected state visible across all pages (dashboard, settings, backups, monitoring) with hatched red bars and badges. Backup guards skip disconnected drives. 22 files changed (1 new: monitor/watchdog.go), ~1500 lines added. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
261 lines
6.2 KiB
Go
261 lines
6.2 KiB
Go
package report
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
|
)
|
|
|
|
// BuildReport collects current state from all subsystems and returns a Report.
|
|
func BuildReport(
|
|
cfg *config.Config,
|
|
stackMgr *stacks.Manager,
|
|
backupMgr *backup.Manager,
|
|
cpuCollector *system.CPUCollector,
|
|
metricsStore *metrics.MetricsStore,
|
|
version string,
|
|
storagePaths []settings.StoragePath,
|
|
) *Report {
|
|
r := &Report{
|
|
Version: 1,
|
|
CustomerID: cfg.Customer.ID,
|
|
CustomerName: cfg.Customer.Name,
|
|
ControllerVersion: version,
|
|
Timestamp: time.Now().UTC(),
|
|
}
|
|
|
|
// Controller URL for hub callbacks (self-update trigger, etc.)
|
|
if cfg.Customer.Domain != "" {
|
|
r.ControllerURL = fmt.Sprintf("https://felhom.%s", cfg.Customer.Domain)
|
|
}
|
|
|
|
// System info
|
|
staticInfo := metrics.GetStaticInfo()
|
|
hddPath := cfg.Paths.HDDPath
|
|
if len(storagePaths) > 0 {
|
|
hddPath = storagePaths[0].Path
|
|
}
|
|
sysInfo := system.GetInfo(hddPath, cpuCollector)
|
|
|
|
r.System = SystemReport{
|
|
Hostname: staticInfo.Hostname,
|
|
OS: staticInfo.OS,
|
|
Kernel: staticInfo.Kernel,
|
|
CPUModel: staticInfo.CPUModel,
|
|
CPUCores: staticInfo.CPUCores,
|
|
UptimeSeconds: staticInfo.UptimeSeconds,
|
|
CPUPercent: sysInfo.CPUPercent,
|
|
MemoryTotalMB: sysInfo.TotalMemMB,
|
|
MemoryUsedMB: sysInfo.UsedMemMB,
|
|
MemoryPercent: sysInfo.MemPercent,
|
|
TemperatureCelsius: sysInfo.TemperatureCelsius,
|
|
LoadAvg1: sysInfo.LoadAvg1,
|
|
LoadAvg5: sysInfo.LoadAvg5,
|
|
LoadAvg15: sysInfo.LoadAvg15,
|
|
}
|
|
|
|
// Storage — root filesystem + all registered storage paths
|
|
r.Storage = []StorageReport{
|
|
{Mount: "/", Label: "SSD", TotalGB: sysInfo.DiskTotalGB, UsedGB: sysInfo.DiskUsedGB, Percent: sysInfo.DiskPercent},
|
|
}
|
|
for _, sp := range storagePaths {
|
|
if sp.Disconnected {
|
|
r.Storage = append(r.Storage, StorageReport{
|
|
Mount: sp.Path,
|
|
Label: sp.Label,
|
|
Disconnected: true,
|
|
})
|
|
continue
|
|
}
|
|
di := system.GetDiskUsage(sp.Path)
|
|
if di == nil {
|
|
continue
|
|
}
|
|
r.Storage = append(r.Storage, StorageReport{
|
|
Mount: sp.Path,
|
|
Label: sp.Label,
|
|
TotalGB: di.TotalGB,
|
|
UsedGB: di.UsedGB,
|
|
Percent: di.UsedPercent,
|
|
})
|
|
}
|
|
|
|
// Containers
|
|
r.Containers = buildContainerReport(stackMgr, metricsStore)
|
|
|
|
// Backup
|
|
r.Backup = buildBackupReport(cfg, backupMgr)
|
|
|
|
// Health
|
|
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, storagePaths)
|
|
r.Health = HealthReport{
|
|
Status: healthReport.Status,
|
|
Issues: healthReport.Issues,
|
|
Warnings: healthReport.Warnings,
|
|
}
|
|
if r.Health.Issues == nil {
|
|
r.Health.Issues = []string{}
|
|
}
|
|
if r.Health.Warnings == nil {
|
|
r.Health.Warnings = []string{}
|
|
}
|
|
|
|
// Stacks
|
|
r.Stacks = buildStacksReport(stackMgr)
|
|
|
|
return r
|
|
}
|
|
|
|
func buildContainerReport(stackMgr *stacks.Manager, metricsStore *metrics.MetricsStore) ContainerReport {
|
|
cr := ContainerReport{}
|
|
|
|
allStacks := stackMgr.GetStacks()
|
|
|
|
// Build a map of container stats from metrics store
|
|
statsMap := make(map[string]metrics.ContainerCurrentStats)
|
|
if metricsStore != nil {
|
|
if stats, err := metricsStore.QueryContainerSummary(); err == nil {
|
|
for _, s := range stats {
|
|
statsMap[s.ContainerName] = s
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, s := range allStacks {
|
|
if !s.Deployed {
|
|
continue
|
|
}
|
|
for _, c := range s.Containers {
|
|
cr.Total++
|
|
switch c.State {
|
|
case stacks.StateRunning, stacks.StateStarting:
|
|
cr.Running++
|
|
case stacks.StateUnhealthy:
|
|
cr.Unhealthy++
|
|
cr.Running++ // unhealthy containers are still running
|
|
default:
|
|
cr.Stopped++
|
|
}
|
|
|
|
detail := ContainerDetailReport{
|
|
Name: c.Name,
|
|
State: string(c.State),
|
|
}
|
|
if cs, ok := statsMap[c.Name]; ok {
|
|
detail.CPUPercent = cs.CPUPercent
|
|
detail.MemoryMB = cs.MemUsageMB
|
|
}
|
|
cr.List = append(cr.List, detail)
|
|
}
|
|
}
|
|
|
|
if cr.List == nil {
|
|
cr.List = []ContainerDetailReport{}
|
|
}
|
|
|
|
return cr
|
|
}
|
|
|
|
func buildBackupReport(cfg *config.Config, backupMgr *backup.Manager) BackupReport {
|
|
br := BackupReport{
|
|
Enabled: cfg.Backup.Enabled,
|
|
}
|
|
|
|
if backupMgr == nil {
|
|
return br
|
|
}
|
|
|
|
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
|
|
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
|
|
status := backupMgr.GetFullStatus(nextDBDump, nextBackup)
|
|
|
|
if status.LastDBDump != nil {
|
|
t := status.LastDBDump.LastRun
|
|
br.LastDBDump = &t
|
|
}
|
|
if status.LastBackup != nil {
|
|
t := status.LastBackup.LastRun
|
|
br.LastSnapshot = &t
|
|
}
|
|
if status.RepoStats != nil {
|
|
br.SnapshotCount = status.RepoStats.SnapshotCount
|
|
br.RepoSizeMB = parseSizeToMB(status.RepoStats.TotalSize)
|
|
}
|
|
if !status.LastCheckTime.IsZero() {
|
|
t := status.LastCheckTime
|
|
br.LastIntegrityCheck = &t
|
|
}
|
|
br.IntegrityOK = status.LastCheckOK
|
|
|
|
// Include restic password for hub-side disaster recovery
|
|
if pw, err := backupMgr.GetResticPassword(); err == nil {
|
|
br.ResticPassword = pw
|
|
}
|
|
|
|
return br
|
|
}
|
|
|
|
func buildStacksReport(stackMgr *stacks.Manager) StacksReport {
|
|
sr := StacksReport{}
|
|
allStacks := stackMgr.GetStacks()
|
|
|
|
for _, s := range allStacks {
|
|
if s.Protected {
|
|
continue
|
|
}
|
|
if s.Deployed {
|
|
sr.Deployed = append(sr.Deployed, s.Name)
|
|
} else {
|
|
sr.Available = append(sr.Available, s.Name)
|
|
}
|
|
}
|
|
|
|
if sr.Deployed == nil {
|
|
sr.Deployed = []string{}
|
|
}
|
|
if sr.Available == nil {
|
|
sr.Available = []string{}
|
|
}
|
|
|
|
return sr
|
|
}
|
|
|
|
// parseSizeToMB parses a formatted size string like "1.5 GB", "512.0 MB" into MB.
|
|
func parseSizeToMB(s string) int64 {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0
|
|
}
|
|
|
|
parts := strings.Fields(s)
|
|
if len(parts) != 2 {
|
|
return 0
|
|
}
|
|
|
|
val, err := strconv.ParseFloat(parts[0], 64)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
switch strings.ToUpper(parts[1]) {
|
|
case "GB":
|
|
return int64(val * 1024)
|
|
case "MB":
|
|
return int64(val)
|
|
case "KB":
|
|
return int64(val / 1024)
|
|
default:
|
|
return int64(val)
|
|
}
|
|
}
|