Files
felhom.eu/hub/cmd/hub/main.go
T
admin 079a2cdd08 hub v0.29.0: Day-0 artifact manifest — version dropdowns + auto-derived sha
Operator picks a version from a Gitea-populated dropdown; the hub reads that
version's sha256 from Gitea itself (files-metadata API, no artifact download) and
vouches it — no hand-copied checksums. New internal/gitea read-only client
(ListVersions + FileSHA256, unit-tested). Configuration UI: version <select>s +
read-only sha display; handleSetArtifacts derives the sha authoritatively and
refuses the save on a Gitea lookup failure. Degrades to manual text entry without
registry creds. go build/vet/test clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-01 09:02:13 +02:00

506 lines
18 KiB
Go

package main
import (
"context"
"flag"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/api"
"gitea.dooplex.hu/admin/felhom-hub/internal/assets"
"gitea.dooplex.hu/admin/felhom-hub/internal/gitea"
"gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay"
"gitea.dooplex.hu/admin/felhom-hub/internal/monitor"
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
"gitea.dooplex.hu/admin/felhom-hub/internal/web"
"gopkg.in/yaml.v3"
)
var (
Version = "dev"
BuildTime = "unknown"
)
// Config is the hub configuration loaded from hub.yaml.
type Config struct {
Auth struct {
PasswordHash string `yaml:"password_hash"`
} `yaml:"auth"`
API struct {
ReportAPIKey string `yaml:"report_api_key"`
} `yaml:"api"`
Notifications struct {
ResendAPIKey string `yaml:"resend_api_key"`
FromEmail string `yaml:"from_email"`
OperatorEmail string `yaml:"operator_email"`
OperatorEnabled bool `yaml:"operator_enabled"`
} `yaml:"notifications"`
Retention struct {
MaxDays int `yaml:"max_days"`
PruneSchedule string `yaml:"prune_schedule"`
} `yaml:"retention"`
Alerting struct {
StaleThreshold string `yaml:"stale_threshold"`
// Host root-fs disk-pressure thresholds (percent). Empty/0/invalid → defaults 90/95
// (normalizeDiskThresholds). Seed-only config, like stale_threshold.
HostDiskWarnPercent float64 `yaml:"host_disk_warn_percent"`
HostDiskCritPercent float64 `yaml:"host_disk_crit_percent"`
// Per-storage-target fill thresholds (percent). Empty/0/invalid → defaults 90/95. Independent of
// the host-root thresholds above so a single storage's policy can be tuned separately.
StorageFillWarnPercent float64 `yaml:"storage_fill_warn_percent"`
StorageFillCritPercent float64 `yaml:"storage_fill_crit_percent"`
} `yaml:"alerting"`
Registry struct {
Image string `yaml:"image"`
Username string `yaml:"username"`
Token string `yaml:"token"`
CheckInterval string `yaml:"check_interval"`
TemplateInterval string `yaml:"template_interval"`
} `yaml:"registry"`
Server struct {
Listen string `yaml:"listen"`
DataDir string `yaml:"data_dir"`
} `yaml:"server"`
ControllerUpdates struct {
// DefaultMinVersion is the global controller-version FLOOR fallback (Phase 2 managed updates):
// the minimum controller version any customer's box auto-updates to, unless overridden
// per-customer or via the operator UI (hub_settings). Empty = no global floor. Env-overridable
// with DEFAULT_MIN_CONTROLLER_VERSION.
DefaultMinVersion string `yaml:"default_min_version"`
} `yaml:"controller_updates"`
Mail MailConfig `yaml:"mail"`
}
// MailConfig tunes the app-email passthrough (POST /api/v1/mail).
type MailConfig struct {
// PerCustomerPerMinute caps a single customer's forwarded messages per minute (abuse
// containment — one box can't drain the shared Resend quota). Default 30.
PerCustomerPerMinute int `yaml:"per_customer_per_minute"`
// FromDomains is the From-header allowlist (backstop; Resend is the final backstop).
// Default ["felhom.eu"].
FromDomains []string `yaml:"from_domains"`
}
func (m MailConfig) effectivePerMinute() int {
if m.PerCustomerPerMinute <= 0 {
return 30
}
return m.PerCustomerPerMinute
}
func (m MailConfig) effectiveFromDomains() []string {
if len(m.FromDomains) == 0 {
return []string{"felhom.eu"}
}
return m.FromDomains
}
func main() {
configPath := flag.String("config", "/etc/felhom-hub/hub.yaml", "Path to configuration file")
showVersion := flag.Bool("version", false, "Show version and exit")
flag.Parse()
if *showVersion {
fmt.Printf("felhom-hub %s (built %s)\n", Version, BuildTime)
os.Exit(0)
}
logger := log.New(os.Stdout, "", log.LstdFlags)
logger.Printf("[INFO] felhom-hub %s starting", Version)
// Load config
cfg := loadConfig(*configPath, logger)
// Environment variable overrides (for k8s Secrets)
if v := os.Getenv("REGISTRY_USERNAME"); v != "" {
cfg.Registry.Username = v
}
if v := os.Getenv("REGISTRY_TOKEN"); v != "" {
cfg.Registry.Token = v
}
if v := os.Getenv("DEFAULT_MIN_CONTROLLER_VERSION"); v != "" {
cfg.ControllerUpdates.DefaultMinVersion = v
}
// Resend API key is sourced from Secret/resend-api (env RESEND_API_KEY), never from the
// committed ConfigMap — see documentation/runbooks/secrets.md. The ConfigMap field stays
// empty as a placeholder; this override is the live source.
if v := os.Getenv("RESEND_API_KEY"); v != "" {
cfg.Notifications.ResendAPIKey = v
}
// Ensure data dir exists
os.MkdirAll(cfg.Server.DataDir, 0755)
// Initialize store
dbPath := filepath.Join(cfg.Server.DataDir, "hub.db")
dataStore, err := store.New(dbPath, logger)
if err != nil {
logger.Fatalf("[FATAL] Failed to initialize store: %v", err)
}
defer dataStore.Close()
logger.Printf("[INFO] Database opened at %s", dbPath)
// Phase 2 managed updates: seed the global controller-version floor fallback (config/env). A
// hub_settings row set via the operator UI overrides this at runtime.
dataStore.SetDefaultMinControllerVersion(cfg.ControllerUpdates.DefaultMinVersion)
if cfg.ControllerUpdates.DefaultMinVersion != "" {
logger.Printf("[INFO] Default controller-version floor: %s", cfg.ControllerUpdates.DefaultMinVersion)
}
// BUNDLE slice: seed the Day-0 artifact manifest from env (ARTIFACT_AGENT_VERSION /
// ARTIFACT_AGENT_SHA256 / ARTIFACT_GOLDEN_VERSION / ARTIFACT_GOLDEN_SHA256). The operator UI is the
// primary editor, but the UI is password-gated, so env-seeding is the same escape hatch the floor
// uses. Seed ONLY fields the DB doesn't already have, so a UI edit sticks across restarts and a
// fresh install still gets values from the deployment env.
{
m := dataStore.GetArtifactManifest()
changed := false
if m.AgentVersion == "" {
if v := os.Getenv("ARTIFACT_AGENT_VERSION"); v != "" {
m.AgentVersion = v
changed = true
}
}
if m.AgentSHA256 == "" {
if v := os.Getenv("ARTIFACT_AGENT_SHA256"); v != "" {
m.AgentSHA256 = v
changed = true
}
}
if m.GoldenVersion == "" {
if v := os.Getenv("ARTIFACT_GOLDEN_VERSION"); v != "" {
m.GoldenVersion = v
changed = true
}
}
if m.GoldenSHA256 == "" {
if v := os.Getenv("ARTIFACT_GOLDEN_SHA256"); v != "" {
m.GoldenSHA256 = v
changed = true
}
}
if changed {
if err := dataStore.SetArtifactManifest(m); err != nil {
logger.Printf("[WARN] Failed to seed artifact manifest from env: %v", err)
} else {
logger.Printf("[INFO] Artifact manifest seeded from env: agent=%s golden=%s", m.AgentVersion, m.GoldenVersion)
}
}
}
// Parse stale threshold
staleThreshold, err := time.ParseDuration(cfg.Alerting.StaleThreshold)
if err != nil {
staleThreshold = 30 * time.Minute
}
// Background context for all goroutines
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Initialize template fetcher for customer config generation
var templateFetcher *web.TemplateFetcher
if cfg.Registry.Username != "" && cfg.Registry.Token != "" {
templateInterval, err := time.ParseDuration(cfg.Registry.TemplateInterval)
if err != nil {
templateInterval = 1 * time.Hour
}
templateFetcher = web.NewTemplateFetcher(cfg.Registry.Username, cfg.Registry.Token, templateInterval, logger)
go templateFetcher.Run(ctx)
logger.Printf("[INFO] Template fetcher started (every %s)", cfg.Registry.TemplateInterval)
}
// Initialize asset manager (PVC storage with image seed)
assetsDir := filepath.Join(cfg.Server.DataDir, "assets")
assetsMgr := assets.New(assetsDir, "/usr/share/felhom/assets-seed", logger)
// Initialize handlers — pass templateFetcher as interface (nil-safe)
var templateProvider api.ConfigTemplateProvider
if templateFetcher != nil {
templateProvider = templateFetcher
}
apiHandler := api.New(dataStore, cfg.API.ReportAPIKey, cfg.Notifications.ResendAPIKey, cfg.Notifications.FromEmail, templateProvider, logger)
apiHandler.SetAssetManager(assetsMgr)
// Initialize notification dispatcher
dispatcher := notify.NewDispatcher(
dataStore,
cfg.Notifications.ResendAPIKey,
cfg.Notifications.FromEmail,
cfg.Notifications.OperatorEmail,
cfg.Notifications.OperatorEnabled,
logger,
)
apiHandler.SetDispatcher(dispatcher)
// App-email passthrough (POST /api/v1/mail): a customer box's shim forwards a raw message
// here and the hub re-emits it to Resend over SMTP, UNCHANGED (raw passthrough — NOT the
// dispatcher's structured HTTP-API path, which drops inline CID images). The Resend key stays
// hub-side. Wired only when a key is present; otherwise the endpoint returns 503.
if cfg.Notifications.ResendAPIKey != "" {
mailSender := mailrelay.NewResendSMTP(cfg.Notifications.ResendAPIKey)
apiHandler.SetMailRelay(mailSender, cfg.Mail.PerCustomerPerMinute, cfg.Mail.FromDomains)
logger.Printf("[INFO] App-email relay enabled (limit %d/min/customer, From domains %v)", cfg.Mail.effectivePerMinute(), cfg.Mail.effectiveFromDomains())
} else {
logger.Printf("[INFO] App-email relay disabled (no Resend key)")
}
webServer := web.New(dataStore, cfg.Auth.PasswordHash, cfg.API.ReportAPIKey, Version, staleThreshold, logger)
webServer.SetTemplateFetcher(templateFetcher)
webServer.SetAssetManager(assetsMgr)
// Day-0 artifact version dropdowns: let the operator pick a version and have the hub derive the
// sha256 from Gitea (no hand-copied checksums). Reuses the registry creds; degrades to manual text
// entry when they're absent.
if cfg.Registry.Username != "" && cfg.Registry.Token != "" {
webServer.SetGiteaClient(gitea.New("https://gitea.dooplex.hu", "admin", cfg.Registry.Username, cfg.Registry.Token))
logger.Printf("[INFO] Gitea artifact browser enabled (Day-0 version dropdowns)")
}
// Build HTTP mux
mux := http.NewServeMux()
// Health check endpoint — bypasses auth (for k8s probes)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
if err := dataStore.Ping(); err != nil {
w.WriteHeader(http.StatusServiceUnavailable)
w.Write([]byte("db unhealthy"))
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
// API routes (no dashboard auth for report ingest)
mux.Handle("/api/v1/", apiHandler)
// Web routes (auth required)
if cfg.Auth.PasswordHash != "" {
mux.Handle("/", webServer.RequireAuth(http.HandlerFunc(webServer.ServeHTTP)))
} else {
mux.Handle("/", http.HandlerFunc(webServer.ServeHTTP))
}
// Start HTTP server
server := &http.Server{
Addr: cfg.Server.Listen,
Handler: mux,
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
}
// Initialize version checker for controller image registry
var versionChecker *web.VersionChecker
if cfg.Registry.Username != "" && cfg.Registry.Token != "" {
checkInterval, err := time.ParseDuration(cfg.Registry.CheckInterval)
if err != nil {
checkInterval = 6 * time.Hour
}
versionChecker = web.NewVersionChecker(cfg.Registry.Image, cfg.Registry.Username, cfg.Registry.Token, checkInterval, logger)
go versionChecker.Run(ctx)
logger.Printf("[INFO] Registry version checker started (every %s)", cfg.Registry.CheckInterval)
} else {
logger.Printf("[INFO] Registry version checker disabled (no credentials configured)")
}
webServer.SetVersionChecker(versionChecker)
// Advertise latest on the controller report ACK (Phase 2). Guard the nil case so we don't store a
// non-nil interface wrapping a nil *VersionChecker.
if versionChecker != nil {
apiHandler.SetLatestVersionProvider(versionChecker)
}
// Session cleanup — removes expired sessions every hour
go webServer.CleanupSessions(ctx)
// Prune on startup, then daily at configured time (default 04:30)
if cfg.Retention.MaxDays > 0 {
pruneAll(dataStore, cfg.Retention.MaxDays, logger)
go scheduleDaily(ctx, "prune", cfg.Retention.PruneSchedule, func() {
pruneAll(dataStore, cfg.Retention.MaxDays, logger)
}, logger)
}
// Staleness checker — runs every 60s
stalenessChecker := monitor.NewStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger)
// v0.7.0: host-domain dead-man's-switch (sibling; the controller checker above is
// unchanged and keeps running until the slice-10 cutover). Same 60s cadence.
hostStalenessChecker := monitor.NewHostStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger)
// v0.44.0-agent: operator alert when an agent reports a degraded privileged capability (a missing
// `sudo -n` grant — the 2026-06-28 cutover class). Same 60s sweep + transition/cooldown plumbing.
hostCapabilityChecker := monitor.NewHostCapabilityChecker(dataStore, dispatcher.ProcessEvent, logger)
// agent-v0.48.0: proactive fleet-wide agent-re-key detection — alert when a host's reported
// local-API leaf fingerprint changes (independent of the controller channel-check). Same 60s sweep.
hostLeafChecker := monitor.NewHostLeafChecker(dataStore, dispatcher.ProcessEvent, logger)
// v0.23.0: HOST root-fs disk-pressure alert (the silent vzdump-on-root-fills class). Reads the host
// root disk_percent the agent reports; born/persistent (a disk already full at hub restart alerts on
// cycle 1). Distinct event types from the controller's GUEST disk_warning/disk_critical. Same 60s sweep.
hostDiskChecker := monitor.NewHostDiskChecker(dataStore, cfg.Alerting.HostDiskWarnPercent, cfg.Alerting.HostDiskCritPercent, dispatcher.ProcessEvent, logger)
// v0.25.0: PER-STORAGE worst-fill alert — generalizes the host-root check to any reported storage
// target (dump/backup volume, data drive, lvmthin pool, PBS datastore). Born/persistent; excludes the
// root-backed builtin (hostDiskChecker owns root, no double-alert); emits natural `critical`. Same sweep.
storageFillChecker := monitor.NewStorageFillChecker(dataStore, cfg.Alerting.StorageFillWarnPercent, cfg.Alerting.StorageFillCritPercent, dispatcher.ProcessEvent, logger)
go func() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
stalenessChecker.Check()
hostStalenessChecker.Check()
hostCapabilityChecker.Check()
hostLeafChecker.Check()
hostDiskChecker.Check()
storageFillChecker.Check()
}
}
}()
// Backup deadline checker — runs daily at 05:00 Budapest
go scheduleDaily(ctx, "deadline-check", "05:00", func() {
monitor.CheckBackupDeadlines(dataStore, stalenessChecker, dispatcher.ProcessEvent, logger)
}, logger)
// Signal handling
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigCh
logger.Printf("[INFO] Received signal %v, shutting down...", sig)
cancel()
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second)
defer shutdownCancel()
if err := server.Shutdown(shutdownCtx); err != nil {
logger.Printf("[ERROR] HTTP server shutdown error: %v", err)
}
}()
logger.Printf("[INFO] Listening on %s", cfg.Server.Listen)
if err := server.ListenAndServe(); err != http.ErrServerClosed {
logger.Fatalf("[FATAL] HTTP server error: %v", err)
}
logger.Println("[INFO] felhom-hub stopped")
}
func loadConfig(path string, logger *log.Logger) *Config {
cfg := &Config{}
// Defaults
cfg.Server.Listen = ":8080"
cfg.Server.DataDir = "/data"
cfg.Retention.MaxDays = 90
cfg.Retention.PruneSchedule = "04:30"
cfg.Alerting.StaleThreshold = "30m"
data, err := os.ReadFile(path)
if err != nil {
logger.Printf("[WARN] Config file not found at %s, using defaults", path)
return cfg
}
if err := yaml.Unmarshal(data, cfg); err != nil {
logger.Printf("[WARN] Failed to parse config: %v, using defaults", err)
return cfg
}
// Apply defaults for zero values
if cfg.Server.Listen == "" {
cfg.Server.Listen = ":8080"
}
if cfg.Server.DataDir == "" {
cfg.Server.DataDir = "/data"
}
if cfg.Retention.MaxDays == 0 {
cfg.Retention.MaxDays = 90
}
if cfg.Alerting.StaleThreshold == "" {
cfg.Alerting.StaleThreshold = "30m"
}
if cfg.Notifications.FromEmail == "" {
cfg.Notifications.FromEmail = "monitoring@felhom.eu"
}
if cfg.Registry.Image == "" {
cfg.Registry.Image = "gitea.dooplex.hu/admin/felhom-controller"
}
if cfg.Registry.CheckInterval == "" {
cfg.Registry.CheckInterval = "6h"
}
if cfg.Registry.TemplateInterval == "" {
cfg.Registry.TemplateInterval = "1h"
}
return cfg
}
// scheduleDaily runs fn once daily at the given "HH:MM" time in Europe/Budapest.
// It blocks until ctx is cancelled.
func scheduleDaily(ctx context.Context, name, timeStr string, fn func(), logger *log.Logger) {
budapest, err := time.LoadLocation("Europe/Budapest")
if err != nil {
budapest = time.FixedZone("CET", 3600)
}
hour, min := parseHM(timeStr)
for {
now := time.Now().In(budapest)
next := time.Date(now.Year(), now.Month(), now.Day(), hour, min, 0, 0, budapest)
if !next.After(now) {
next = next.Add(24 * time.Hour)
}
delay := time.Until(next)
logger.Printf("[INFO] %s: next run at %s (in %s)", name, next.Format("2006-01-02 15:04 MST"), delay.Round(time.Second))
select {
case <-ctx.Done():
return
case <-time.After(delay):
fn()
}
}
}
// parseHM parses "HH:MM" into hour and minute. Returns 0, 0 on invalid input.
func parseHM(s string) (int, int) {
var h, m int
if _, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil {
return 0, 0
}
return h, m
}
func pruneAll(s *store.Store, maxDays int, logger *log.Logger) {
if deleted, err := s.Prune(maxDays); err != nil {
logger.Printf("[WARN] Prune reports failed: %v", err)
} else if deleted > 0 {
logger.Printf("[INFO] Pruned %d old report rows", deleted)
}
if deleted, err := s.PruneEvents(maxDays); err != nil {
logger.Printf("[WARN] Prune events failed: %v", err)
} else if deleted > 0 {
logger.Printf("[INFO] Pruned %d old event rows", deleted)
}
if n, err := s.PruneAppTelemetry(time.Now().Add(-90 * 24 * time.Hour)); err != nil {
logger.Printf("[ERROR] Prune app telemetry: %v", err)
} else if n > 0 {
logger.Printf("[INFO] Pruned %d old app telemetry rows", n)
}
if n, err := s.PruneStaleIssues(time.Now().Add(-30 * 24 * time.Hour)); err != nil {
logger.Printf("[ERROR] Prune stale issues: %v", err)
} else if n > 0 {
logger.Printf("[INFO] Pruned %d stale app issues", n)
}
}