felhom-controller/controller/internal/monitor/healthcheck.go

package monitor

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
	"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
)

// HealthReport contains the results of a system health check.
type HealthReport struct {
	Status    string   // "ok", "warn", "fail"
	Issues    []string // critical problems
	Warnings  []string // non-critical warnings
	Info      []string // informational items
	Timestamp time.Time
}

// RunHealthCheck runs system checks and returns a diagnostic report.
func RunHealthCheck(cfg *config.Config, cpuCollector *system.CPUCollector, storagePaths []settings.StoragePath, smb settings.SMBSettings, logger *log.Logger) *HealthReport {
	report := &HealthReport{
		Status:    "ok",
		Timestamp: time.Now(),
	}

	debug := cfg.Logging.Level == "debug" && logger != nil

	hddPath := cfg.Paths.HDDPath
	if len(storagePaths) > 0 {
		hddPath = storagePaths[0].Path
	}
	sysInfo := system.GetInfo(hddPath, cpuCollector)

	if debug {
		logger.Printf("[DEBUG] [monitor] Raw values: disk=%.1f%%, hdd=%.1f%% (configured=%v), mem=%.1f%% (%dMB/%dMB), cpu=%.1f%%, temp=%.1f°C (%s)",
			sysInfo.DiskPercent, sysInfo.HDDPercent, sysInfo.HDDConfigured,
			sysInfo.MemPercent, sysInfo.UsedMemMB, sysInfo.TotalMemMB,
			sysInfo.CPUPercent, sysInfo.TemperatureCelsius, sysInfo.TemperatureSource)
	}

	// 1. Disk usage (SSD). NOTE (storage-split): sysInfo.DiskPercent statfs's the controller
	// container's "/", whose overlay upperdir lives on the guest's /var/lib/docker volume — so this
	// IS the Docker-data volume guard (post-split it's the dedicated data volume; pre-split it's the
	// rootfs — either way it's wherever Docker's data-root lives). Warn at 80% / crit at 90% used
	// trips ABOVE the prevention layer's 10%-free reserved buffer, so the customer is warned before
	// the deploy gate even engages.
	if sysInfo.DiskPercent > 0 {
		if sysInfo.DiskPercent >= float64(cfg.Monitoring.Thresholds.DiskCritPercent) {
			report.Issues = append(report.Issues, fmt.Sprintf("SSD disk usage critical: %.0f%%", sysInfo.DiskPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Disk (SSD) threshold breached: %.0f%% (limit: %d%%)", sysInfo.DiskPercent, cfg.Monitoring.Thresholds.DiskCritPercent)
			}
			if debug {
				logger.Printf("[DEBUG] [monitor] SSD disk: CRITICAL (%.0f%% >= %d%%)", sysInfo.DiskPercent, cfg.Monitoring.Thresholds.DiskCritPercent)
			}
		} else if sysInfo.DiskPercent >= float64(cfg.Monitoring.Thresholds.DiskWarnPercent) {
			report.Warnings = append(report.Warnings, fmt.Sprintf("SSD disk usage high: %.0f%%", sysInfo.DiskPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Disk (SSD) threshold breached: %.0f%% (limit: %d%%)", sysInfo.DiskPercent, cfg.Monitoring.Thresholds.DiskWarnPercent)
			}
			if debug {
				logger.Printf("[DEBUG] [monitor] SSD disk: WARN (%.0f%% >= %d%%)", sysInfo.DiskPercent, cfg.Monitoring.Thresholds.DiskWarnPercent)
			}
		} else {
			report.Info = append(report.Info, fmt.Sprintf("SSD: %.0f%% used", sysInfo.DiskPercent))
			if debug {
				logger.Printf("[DEBUG] [monitor] SSD disk: OK (%.0f%%)", sysInfo.DiskPercent)
			}
		}
	}

	// HDD disk usage
	if sysInfo.HDDConfigured && sysInfo.HDDPercent > 0 {
		if sysInfo.HDDPercent >= float64(cfg.Monitoring.Thresholds.DiskCritPercent) {
			report.Issues = append(report.Issues, fmt.Sprintf("HDD disk usage critical: %.0f%%", sysInfo.HDDPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Disk (HDD) threshold breached: %.0f%% (limit: %d%%)", sysInfo.HDDPercent, cfg.Monitoring.Thresholds.DiskCritPercent)
			}
		} else if sysInfo.HDDPercent >= float64(cfg.Monitoring.Thresholds.DiskWarnPercent) {
			report.Warnings = append(report.Warnings, fmt.Sprintf("HDD disk usage high: %.0f%%", sysInfo.HDDPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Disk (HDD) threshold breached: %.0f%% (limit: %d%%)", sysInfo.HDDPercent, cfg.Monitoring.Thresholds.DiskWarnPercent)
			}
		}
	}

	// 2. Memory usage
	if sysInfo.MemPercent > 0 {
		if sysInfo.MemPercent >= float64(cfg.Monitoring.Thresholds.MemoryWarnPercent) {
			report.Warnings = append(report.Warnings, fmt.Sprintf("Memory usage high: %.0f%%", sysInfo.MemPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Memory threshold breached: %.0f%% (limit: %d%%)", sysInfo.MemPercent, cfg.Monitoring.Thresholds.MemoryWarnPercent)
			}
			if debug {
				logger.Printf("[DEBUG] [monitor] Memory: WARN (%.0f%% >= %d%%)", sysInfo.MemPercent, cfg.Monitoring.Thresholds.MemoryWarnPercent)
			}
		} else {
			report.Info = append(report.Info, fmt.Sprintf("Memory: %.0f%% used", sysInfo.MemPercent))
			if debug {
				logger.Printf("[DEBUG] [monitor] Memory: OK (%.0f%%)", sysInfo.MemPercent)
			}
		}
	}

	// 3. CPU usage
	if sysInfo.CPUPercent > 0 {
		if sysInfo.CPUPercent >= float64(cfg.Monitoring.Thresholds.CPUWarnPercent) {
			report.Warnings = append(report.Warnings, fmt.Sprintf("CPU usage high: %.0f%%", sysInfo.CPUPercent))
			if logger != nil {
				logger.Printf("[WARN] [monitor] CPU threshold breached: %.0f%% (limit: %d%%)", sysInfo.CPUPercent, cfg.Monitoring.Thresholds.CPUWarnPercent)
			}
			if debug {
				logger.Printf("[DEBUG] [monitor] CPU: WARN (%.0f%% >= %d%%)", sysInfo.CPUPercent, cfg.Monitoring.Thresholds.CPUWarnPercent)
			}
		} else {
			report.Info = append(report.Info, fmt.Sprintf("CPU: %.0f%%", sysInfo.CPUPercent))
			if debug {
				logger.Printf("[DEBUG] [monitor] CPU: OK (%.0f%%)", sysInfo.CPUPercent)
			}
		}
	}

	// 4. Temperature
	if sysInfo.TemperatureCelsius > 0 {
		if sysInfo.TemperatureCelsius >= float64(cfg.Monitoring.Thresholds.TemperatureWarnCelsius) {
			report.Warnings = append(report.Warnings, fmt.Sprintf("Temperature high: %.0f°C (%s)", sysInfo.TemperatureCelsius, sysInfo.TemperatureSource))
			if logger != nil {
				logger.Printf("[WARN] [monitor] Temperature threshold breached: %.0f°C (limit: %d°C)", sysInfo.TemperatureCelsius, cfg.Monitoring.Thresholds.TemperatureWarnCelsius)
			}
			if debug {
				logger.Printf("[DEBUG] [monitor] Temperature: WARN (%.0f°C >= %d°C)", sysInfo.TemperatureCelsius, cfg.Monitoring.Thresholds.TemperatureWarnCelsius)
			}
		} else {
			report.Info = append(report.Info, fmt.Sprintf("Temperature: %.0f°C", sysInfo.TemperatureCelsius))
			if debug {
				logger.Printf("[DEBUG] [monitor] Temperature: OK (%.0f°C)", sysInfo.TemperatureCelsius)
			}
		}
	}

	// 5. Docker health
	if err := checkDocker(); err != nil {
		report.Issues = append(report.Issues, fmt.Sprintf("Docker: %v", err))
		if debug {
			logger.Printf("[DEBUG] [monitor] Docker daemon: FAIL (%v)", err)
		}
	} else {
		report.Info = append(report.Info, "Docker: reachable")
		if debug {
			logger.Printf("[DEBUG] [monitor] Docker daemon: OK")
		}
	}

	// 6. Protected containers (effective set: cloudflared only counts when a tunnel token is
	// configured, so a LAN-only node doesn't report FAIL forever for a stack it intentionally skips).
	protected := EffectiveProtected(cfg, smb)
	if debug {
		logger.Printf("[DEBUG] [monitor] Checking %d protected containers: %v", len(protected), protected)
	}
	missingProtected := checkProtectedContainers(protected)
	for _, name := range missingProtected {
		report.Issues = append(report.Issues, fmt.Sprintf("Protected container not running: %s", name))
	}
	if debug {
		if len(missingProtected) > 0 {
			logger.Printf("[DEBUG] [monitor] Protected containers missing: %v", missingProtected)
		} else {
			logger.Printf("[DEBUG] [monitor] All protected containers running")
		}
	}

	// 7. Storage paths
	storageIssues, storageWarnings := checkStoragePaths(storagePaths)
	report.Issues = append(report.Issues, storageIssues...)
	report.Warnings = append(report.Warnings, storageWarnings...)

	// Determine status
	if len(report.Issues) > 0 {
		report.Status = "fail"
	} else if len(report.Warnings) > 0 {
		report.Status = "warn"
	}

	if logger != nil {
		logger.Printf("[INFO] [monitor] Health check: status=%s", report.Status)
	}

	if debug {
		logger.Printf("[DEBUG] [monitor] Final status: %s (issues=%d, warnings=%d, info=%d)",
			report.Status, len(report.Issues), len(report.Warnings), len(report.Info))
	}

	return report
}

// FormatMessage returns a human-readable summary for healthcheck ping body.
func (r *HealthReport) FormatMessage() string {
	var sb strings.Builder

	sb.WriteString(fmt.Sprintf("Status: %s\n", strings.ToUpper(r.Status)))
	sb.WriteString(fmt.Sprintf("Time: %s\n\n", r.Timestamp.Format("2006-01-02 15:04:05")))

	if len(r.Issues) > 0 {
		sb.WriteString("ISSUES:\n")
		for _, issue := range r.Issues {
			sb.WriteString("  - " + issue + "\n")
		}
		sb.WriteString("\n")
	}

	if len(r.Warnings) > 0 {
		sb.WriteString("WARNINGS:\n")
		for _, w := range r.Warnings {
			sb.WriteString("  - " + w + "\n")
		}
		sb.WriteString("\n")
	}

	if len(r.Info) > 0 {
		sb.WriteString("INFO:\n")
		for _, info := range r.Info {
			sb.WriteString("  - " + info + "\n")
		}
	}

	return sb.String()
}

func checkDocker() error {
	cmd := exec.Command("docker", "info", "--format", "{{.ServerVersion}}")
	out, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("docker not reachable: %v", err)
	}
	if len(strings.TrimSpace(string(out))) == 0 {
		return fmt.Errorf("docker returned empty version")
	}
	return nil
}

// EffectiveProtected returns the protected-container set that actually applies to this node. It is
// the configured cfg.Stacks.Protected minus stacks that are intentionally not deployed here, plus
// the DYNAMIC extras whose deployment depends on customer state rather than config:
//
//   - cloudflared is dropped when no tunnel token is configured (a LAN-only node legitimately runs
//     without it, so it must not be reported as a missing protected container forever);
//   - the samba container is ADDED when network sharing is switched on (R-7b). Sharing is a
//     customer-toggled feature, so it can never appear in the golden controller.yaml — but once it
//     IS on, a dead sharing service is exactly as customer-visible as a dead traefik and must raise
//     the same protected-container issue → alert → Hungarian degradation e-mail. When sharing is
//     off the container is absent from the set, so a box that never enabled it stays quiet.
//
// The bring-up applies the same conditions (stacks.EnsureBaseStack for cloudflared, ensureSamba's
// `if !smb.Enabled { return }` for samba), so detection and deployment agree in both directions.
//
// NOTE: the entries are CONTAINER names (checkProtectedContainers docker-inspects them). For the
// base stacks the container name happens to equal the stack name; for samba it does NOT — the stack
// is „samba" but the container is infra.SambaContainerName — which is why the constant is read here
// rather than the stack name assumed.
func EffectiveProtected(cfg *config.Config, smb settings.SMBSettings) []string {
	out := make([]string, 0, len(cfg.Stacks.Protected)+1)
	for _, name := range cfg.Stacks.Protected {
		if name == "cloudflared" && cfg.Infrastructure.CFTunnelToken == "" {
			continue
		}
		out = append(out, name)
	}
	if smb.Enabled {
		out = append(out, infra.SambaContainerName)
	}
	return out
}

func checkProtectedContainers(protected []string) []string {
	var missing []string
	for _, name := range protected {
		cmd := exec.Command("docker", "inspect", "--format", "{{.State.Running}}", name)
		out, err := cmd.Output()
		if err != nil {
			missing = append(missing, name)
			continue
		}
		if strings.TrimSpace(string(out)) != "true" {
			missing = append(missing, name)
		}
	}
	return missing
}

func checkStoragePaths(paths []settings.StoragePath) (issues, warnings []string) {
	for _, sp := range paths {
		// Skip decommissioned paths — no longer in active use
		if sp.Decommissioned {
			continue
		}

		// Skip disconnected paths — handled by the storage watchdog
		if sp.Disconnected {
			warnings = append(warnings, fmt.Sprintf("Meghajtó leválasztva: %s (%s)", sp.Label, sp.Path))
			continue
		}

		// Path accessible?
		if _, err := os.Stat(sp.Path); err != nil {
			warnings = append(warnings, fmt.Sprintf("Adattároló nem elérhető: %s", sp.Path))
			continue
		}

		// Mount point check — warning, not issue (avoids false FAIL on demo/test environments)
		if !system.IsMountPoint(sp.Path) {
			warnings = append(warnings, fmt.Sprintf(
				"Az adattároló (%s) nem külön meghajtón van — az adatok a rendszermeghajtóra íródnak", sp.Path))
		}

		// Disk usage
		if di := system.GetDiskUsage(sp.Path); di != nil {
			if di.UsedPercent >= 95 {
				issues = append(issues, fmt.Sprintf("Adattároló majdnem megtelt: %s (%.0f%%)", sp.Path, di.UsedPercent))
			} else if di.UsedPercent >= 90 {
				warnings = append(warnings, fmt.Sprintf("Adattároló használat magas: %s (%.0f%%)", sp.Path, di.UsedPercent))
			}
		}
	}
	return
}