package main import ( "context" "flag" "fmt" "log" "net/http" "os" "os/signal" "path/filepath" "strconv" "strings" "syscall" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/accesslog" "gitea.dooplex.hu/admin/felhom-hub/internal/api" "gitea.dooplex.hu/admin/felhom-hub/internal/assets" "gitea.dooplex.hu/admin/felhom-hub/internal/claim" "gitea.dooplex.hu/admin/felhom-hub/internal/dbsnap" "gitea.dooplex.hu/admin/felhom-hub/internal/gitea" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" "gitea.dooplex.hu/admin/felhom-hub/internal/intent" "gitea.dooplex.hu/admin/felhom-hub/internal/mailhold" "gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay" "gitea.dooplex.hu/admin/felhom-hub/internal/monitor" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/offsiteheal" "gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys" "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" "gitea.dooplex.hu/admin/felhom-hub/internal/pbsdrheal" "gitea.dooplex.hu/admin/felhom-hub/internal/poke" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" "gitea.dooplex.hu/admin/felhom-hub/internal/web" "gitea.dooplex.hu/admin/felhom-hub/internal/wgsync" "gopkg.in/yaml.v3" ) var ( Version = "dev" BuildTime = "unknown" ) // Config is the hub configuration loaded from hub.yaml. type Config struct { Auth struct { PasswordHash string `yaml:"password_hash"` } `yaml:"auth"` API struct { ReportAPIKey string `yaml:"report_api_key"` } `yaml:"api"` Notifications struct { ResendAPIKey string `yaml:"resend_api_key"` FromEmail string `yaml:"from_email"` OperatorEmail string `yaml:"operator_email"` OperatorEnabled bool `yaml:"operator_enabled"` } `yaml:"notifications"` Retention struct { MaxDays int `yaml:"max_days"` PruneSchedule string `yaml:"prune_schedule"` } `yaml:"retention"` Alerting struct { StaleThreshold string `yaml:"stale_threshold"` // Host root-fs disk-pressure thresholds (percent). Empty/0/invalid → defaults 90/95 // (normalizeDiskThresholds). Seed-only config, like stale_threshold. HostDiskWarnPercent float64 `yaml:"host_disk_warn_percent"` HostDiskCritPercent float64 `yaml:"host_disk_crit_percent"` // Per-storage-target fill thresholds (percent). Empty/0/invalid → defaults 90/95. Independent of // the host-root thresholds above so a single storage's policy can be tuned separately. StorageFillWarnPercent float64 `yaml:"storage_fill_warn_percent"` StorageFillCritPercent float64 `yaml:"storage_fill_crit_percent"` // Offsite POOL-BOX aggregate thresholds (v0.64.0, R-5). Fill = box used vs capacity (percent); // oversub = Σ(shared soft quotas)/capacity (ratio). Empty/0/invalid → defaults 80/90/2.0 (the // checker normalizes). Threshold VALUES are Claude's encoding of the starter suggestion — Viktor's // ruling pending; these keys are the one-line flip when ruled. OffsiteBoxFillWarnPercent float64 `yaml:"offsite_box_fill_warn_percent"` OffsiteBoxFillCritPercent float64 `yaml:"offsite_box_fill_crit_percent"` OffsiteOversubWarnRatio float64 `yaml:"offsite_oversub_warn_ratio"` // PBS-DR datastore fill thresholds (v0.65.0). Separate keys defaulting to the SAME 80/90 as the // restic pool box, so PBS can be tuned independently later without touching the restic policy. // No oversubscription concept for PBS (namespaces, not quotas) — fill only. PBSDRBoxFillWarnPercent float64 `yaml:"pbsdr_box_fill_warn_percent"` PBSDRBoxFillCritPercent float64 `yaml:"pbsdr_box_fill_crit_percent"` // Consecutive failed 15-minute box reads before the hub reports the store unreachable // (v0.106.0, R-339). 0/invalid → 3 (≈30–45 min). Applies to BOTH the restic pool box and the // PBS-DR datastore — one knob, because the two are the same operator question ("can I still // see the off-site tier") and splitting it would invite them to drift apart. BoxUnreachableWindows int `yaml:"box_unreachable_windows"` } `yaml:"alerting"` Registry struct { Image string `yaml:"image"` Username string `yaml:"username"` Token string `yaml:"token"` CheckInterval string `yaml:"check_interval"` TemplateInterval string `yaml:"template_interval"` } `yaml:"registry"` Server struct { Listen string `yaml:"listen"` DataDir string `yaml:"data_dir"` } `yaml:"server"` ControllerUpdates struct { // DefaultMinVersion is the global controller-version FLOOR fallback (Phase 2 managed updates): // the minimum controller version any customer's box auto-updates to, unless overridden // per-customer or via the operator UI (hub_settings). Empty = no global floor. Env-overridable // with DEFAULT_MIN_CONTROLLER_VERSION. DefaultMinVersion string `yaml:"default_min_version"` } `yaml:"controller_updates"` Mail MailConfig `yaml:"mail"` } // MailConfig tunes the app-email passthrough (POST /api/v1/mail). type MailConfig struct { // PerCustomerPerMinute caps a single customer's forwarded messages per minute (abuse // containment — one box can't drain the shared Resend quota). Default 30. PerCustomerPerMinute int `yaml:"per_customer_per_minute"` // FromDomains is the From-header allowlist (backstop; Resend is the final backstop). // Default ["felhom.eu"]. FromDomains []string `yaml:"from_domains"` } func (m MailConfig) effectivePerMinute() int { if m.PerCustomerPerMinute <= 0 { return 30 } return m.PerCustomerPerMinute } func (m MailConfig) effectiveFromDomains() []string { if len(m.FromDomains) == 0 { return []string{"felhom.eu"} } return m.FromDomains } func main() { configPath := flag.String("config", "/etc/felhom-hub/hub.yaml", "Path to configuration file") showVersion := flag.Bool("version", false, "Show version and exit") // R-879 roll-back: open the sealed box secrets back to plaintext so a pre-R-879 hub (<= 0.137.0) // can run on this database, then exit. Same -config and OFFSITE_SECRET_KEY as the server. unsealBox := flag.Bool("unseal-box-secrets", false, "R-879 roll-back: unseal box API keys, passphrases and PBS tokens to plaintext, then exit") sweepAppLog := flag.String("sweep-app-log-residue", "", "R-244 one-time sweep of DELETED customers' ids in app_log_issues: \"count\" (changes nothing) or \"apply\", then exit") flag.Parse() if *showVersion { fmt.Printf("felhom-hub %s (built %s)\n", Version, BuildTime) os.Exit(0) } logger := log.New(os.Stdout, "", log.LstdFlags) logger.Printf("[INFO] felhom-hub %s starting", Version) // Load config cfg := loadConfig(*configPath, logger) // Environment variable overrides (for k8s Secrets) if v := os.Getenv("REGISTRY_USERNAME"); v != "" { cfg.Registry.Username = v } if v := os.Getenv("REGISTRY_TOKEN"); v != "" { cfg.Registry.Token = v } if v := os.Getenv("DEFAULT_MIN_CONTROLLER_VERSION"); v != "" { cfg.ControllerUpdates.DefaultMinVersion = v } // Resend API key is sourced from Secret/resend-api (env RESEND_API_KEY), never from the // committed ConfigMap — see documentation/runbooks/secrets.md. The ConfigMap field stays // empty as a placeholder; this override is the live source. if v := os.Getenv("RESEND_API_KEY"); v != "" { cfg.Notifications.ResendAPIKey = v } // The operator/global bearer key (api.report_api_key) is sourced from Secret/report-api // (env REPORT_API_KEY) since v0.53.0 — the ConfigMap field is an empty placeholder (the // previously-committed literal is dead once rotated; see the publish-runbook ROTATION // notes). Same pattern as RESEND_API_KEY above. if v := os.Getenv("REPORT_API_KEY"); v != "" { cfg.API.ReportAPIKey = v } // Ensure data dir exists os.MkdirAll(cfg.Server.DataDir, 0755) // Initialize store dbPath := filepath.Join(cfg.Server.DataDir, "hub.db") dataStore, err := store.New(dbPath, logger) if err != nil { logger.Fatalf("[FATAL] Failed to initialize store: %v", err) } defer dataStore.Close() // R-821 / decision 69 (v0.127.0): the off-site sub-account password is sealed at rest with a key // that is NOT in the database (Secret/offsite-secret-key). Without it the hub cannot store, read or // use any sub-account password — provisioning and the key registrar refuse (fail-closed), and the // legacy plaintext rows stay as they are until the key arrives. if *unsealBox { code := runUnsealBoxSecrets(dataStore, logger) dataStore.Close() os.Exit(code) } if *sweepAppLog != "" { // R-244 (S6, decision 201): the operator sees the count first ("count"), then says yes ("apply"). // Only ids the hub no longer knows by config, host or report are touched. Prints ids and counts only. if *sweepAppLog != "count" && *sweepAppLog != "apply" { fmt.Fprintln(os.Stderr, `-sweep-app-log-residue takes "count" or "apply"`) os.Exit(2) } res, err := dataStore.SweepDeletedAppIssueRefs(*sweepAppLog == "apply") dataStore.Close() if err != nil { fmt.Fprintf(os.Stderr, "sweep failed (nothing committed): %v\n", err) os.Exit(1) } totDer, totDel := 0, 0 for _, r := range res { fmt.Printf("%s\tdereferenced=%d\tdeleted=%d\n", r.CustomerID, r.Dereferenced, r.Deleted) totDer, totDel = totDer+r.Dereferenced, totDel+r.Deleted } fmt.Printf("%s: %d deleted customer id(s), %d shared row(s) de-referenced, %d orphan row(s) deleted\n", *sweepAppLog, len(res), totDer, totDel) os.Exit(0) } offsiteKeyReady := false if v := os.Getenv("OFFSITE_SECRET_KEY"); v == "" { logger.Printf("[ERROR] OFFSITE_SECRET_KEY unset — off-site passwords cannot be sealed; provisioning and the key registrar are DISABLED") } else if key, kerr := store.ParseOffsiteSecretKey(v); kerr != nil { logger.Printf("[ERROR] OFFSITE_SECRET_KEY invalid (%v) — provisioning and the key registrar are DISABLED", kerr) } else if kerr := dataStore.SetOffsiteSecretKey(key); kerr != nil { logger.Printf("[ERROR] OFFSITE_SECRET_KEY rejected (%v) — provisioning and the key registrar are DISABLED", kerr) } else { offsiteKeyReady = true if n, serr := dataStore.SealLegacyOffsiteSecrets(); serr != nil { logger.Printf("[ERROR] sealing legacy off-site secrets failed after %d row(s): %v", n, serr) } else { logger.Printf("[INFO] off-site secrets sealed at rest (%d legacy plaintext row(s) sealed now)", n) } // R-133 (v0.135.0): the break-glass console passwords use the same key and the same seal. if n, serr := dataStore.SealLegacyRecoverySecrets(); serr != nil { logger.Printf("[ERROR] sealing legacy console passwords failed after %d row(s): %v", n, serr) } else { logger.Printf("[INFO] console passwords sealed at rest (%d legacy plaintext row(s) sealed now)", n) } // R-879: the box API keys, owner passphrases, controller API keys and PBS-DR tokens — same key, // same seal. A failure part-way is logged, never fatal: boxes still authenticate (hash lookup). if n, serr := dataStore.SealLegacyBoxSecrets(); serr != nil { logger.Printf("[ERROR] sealing legacy box secrets failed after %d value(s): %v", n, serr) } else { logger.Printf("[INFO] box secrets sealed at rest (%d legacy plaintext value(s) sealed now)", n) } } logger.Printf("[INFO] Database opened at %s", dbPath) // Phase 2 managed updates: seed the global controller-version floor fallback (config/env). A // hub_settings row set via the operator UI overrides this at runtime. dataStore.SetDefaultMinControllerVersion(cfg.ControllerUpdates.DefaultMinVersion) if reason, spoiled := store.SpoiledControllerTag(cfg.ControllerUpdates.DefaultMinVersion); spoiled { // R-1085: the config/env default is the one floor no form checks — ResolveManagedFloor holds it at serve // time; say so at start, where whoever set it will look. logger.Printf("[WARN] Default controller-version floor %s is a WITHDRAWN controller build (%s) — it is HELD for every box (R-1085)", cfg.ControllerUpdates.DefaultMinVersion, reason) } if cfg.ControllerUpdates.DefaultMinVersion != "" { logger.Printf("[INFO] Default controller-version floor: %s", cfg.ControllerUpdates.DefaultMinVersion) } // BUNDLE slice: seed the Day-0 artifact manifest from env (ARTIFACT_AGENT_VERSION / // ARTIFACT_AGENT_SHA256 / ARTIFACT_GOLDEN_VERSION / ARTIFACT_GOLDEN_SHA256). The operator UI is the // primary editor, but the UI is password-gated, so env-seeding is the same escape hatch the floor // uses. Seed ONLY fields the DB doesn't already have, so a UI edit sticks across restarts and a // fresh install still gets values from the deployment env. { m := dataStore.GetArtifactManifest() changed := false if m.AgentVersion == "" { if v := os.Getenv("ARTIFACT_AGENT_VERSION"); v != "" { m.AgentVersion = v changed = true } } if m.AgentSHA256 == "" { if v := os.Getenv("ARTIFACT_AGENT_SHA256"); v != "" { m.AgentSHA256 = v changed = true } } if m.GoldenVersion == "" { if v := os.Getenv("ARTIFACT_GOLDEN_VERSION"); v != "" { m.GoldenVersion = v changed = true } } if m.GoldenSHA256 == "" { if v := os.Getenv("ARTIFACT_GOLDEN_SHA256"); v != "" { m.GoldenSHA256 = v changed = true } } if changed { if err := dataStore.SetArtifactManifest(m); err != nil { logger.Printf("[WARN] Failed to seed artifact manifest from env: %v", err) } else { logger.Printf("[INFO] Artifact manifest seeded from env: agent=%s golden=%s", m.AgentVersion, m.GoldenVersion) } } } // Parse stale threshold staleThreshold, err := time.ParseDuration(cfg.Alerting.StaleThreshold) if err != nil { staleThreshold = 30 * time.Minute } // Background context for all goroutines ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Initialize template fetcher for customer config generation var templateFetcher *web.TemplateFetcher if cfg.Registry.Username != "" && cfg.Registry.Token != "" { templateInterval, err := time.ParseDuration(cfg.Registry.TemplateInterval) if err != nil { templateInterval = 1 * time.Hour } templateFetcher = web.NewTemplateFetcher(cfg.Registry.Username, cfg.Registry.Token, templateInterval, logger) go templateFetcher.Run(ctx) logger.Printf("[INFO] Template fetcher started (every %s)", cfg.Registry.TemplateInterval) } // Initialize asset manager (PVC storage with image seed) assetsDir := filepath.Join(cfg.Server.DataDir, "assets") assetsMgr := assets.New(assetsDir, "/usr/share/felhom/assets-seed", logger) // Initialize handlers — pass templateFetcher as interface (nil-safe) var templateProvider api.ConfigTemplateProvider if templateFetcher != nil { templateProvider = templateFetcher } apiHandler := api.New(dataStore, cfg.API.ReportAPIKey, cfg.Notifications.ResendAPIKey, cfg.Notifications.FromEmail, templateProvider, logger) apiHandler.SetAssetManager(assetsMgr) // R-1154: the host-enroll reuse path judges "the old box is gone" by the same threshold as hostStatus and the // staleness checkers (unwired = it never re-issues). Pinned by r1154_wiring_test.go. apiHandler.SetStaleThreshold(staleThreshold) // Initialize notification dispatcher dispatcher := notify.NewDispatcher( dataStore, cfg.Notifications.ResendAPIKey, cfg.Notifications.FromEmail, cfg.Notifications.OperatorEmail, cfg.Notifications.OperatorEnabled, logger, ) apiHandler.SetDispatcher(dispatcher) // MAIL-HOLD (internal/mailhold): while /MAIL-HOLD exists the hub sends NO e-mail on any path — // dispatcher, /notify, app-mail relay. A hub started on a restored database mailed households within a minute // (2026-10-09 restore drill); the marker makes a restored copy start quiet. Lifted by the operator with // POST /configuration/mail-hold/release; held mails are dropped, not re-sent. mailHold := mailhold.New(cfg.Server.DataDir, logger) if mailHold.Held() { logger.Printf("[WARN] MAIL-HOLD: marker present at start (%s) — the hub sends no e-mail until the operator releases it on the Configuration page", mailHold.Path) } dispatcher.SetMailHold(mailHold) apiHandler.SetMailHold(mailHold) // Customer-claim password arc (v0.50.0, F-4): the code engine — the dispatcher delivers the // Hungarian emails, the store holds bcrypt(code) only. Wired into the API (Day-0 issue at // config retrieve, live-box issue + claimed ingest + ACK on report, reset endpoint) and the // web UI (status chip + resend). claimEngine := &claim.Engine{Store: dataStore, Mailer: dispatcher, Logger: logger} apiHandler.SetClaimEngine(claimEngine) // App-email passthrough (POST /api/v1/mail): a customer box's shim forwards a raw message // here and the hub re-emits it to Resend over SMTP, UNCHANGED (raw passthrough — NOT the // dispatcher's structured HTTP-API path, which drops inline CID images). The Resend key stays // hub-side. Wired only when a key is present; otherwise the endpoint returns 503. if cfg.Notifications.ResendAPIKey != "" { mailSender := mailrelay.NewResendSMTP(cfg.Notifications.ResendAPIKey) apiHandler.SetMailRelay(mailSender, cfg.Mail.PerCustomerPerMinute, cfg.Mail.FromDomains) logger.Printf("[INFO] App-email relay enabled (limit %d/min/customer, From domains %v)", cfg.Mail.effectivePerMinute(), cfg.Mail.effectiveFromDomains()) } else { logger.Printf("[INFO] App-email relay disabled (no Resend key)") } webServer := web.New(dataStore, cfg.Auth.PasswordHash, cfg.API.ReportAPIKey, Version, staleThreshold, logger) webServer.SetTemplateFetcher(templateFetcher) webServer.SetAssetManager(assetsMgr) // MAIL-HOLD banner on every operator page + the release button on Configuration. webServer.SetMailHold(mailHold) webServer.SetClaimEngine(claimEngine) // v0.50.0 — Setup-tab claim chip + resend button webServer.SetSelfBindMailer(dispatcher) // v0.66.0 (R-27) — customer self-bind link button (sibling of claim mailer) webServer.SetEventEmitter(dispatcher.ProcessEvent) // v0.135.0 (R-604) — the "floor raise skipped boxes" mail // Day-0 artifact version dropdowns: let the operator pick a version and have the hub derive the // sha256 from Gitea (no hand-copied checksums). Reuses the registry creds; degrades to manual text // entry when they're absent. var giteaClient *gitea.Client // also read by the golden-vouch checker (R-242); nil without registry creds if cfg.Registry.Username != "" && cfg.Registry.Token != "" { // IN-CLUSTER, not the public ingress (v0.100.0). The hub and Gitea share this cluster, and // every one of these is a small metadata call made dozens of times per Configuration page // load. Measured on DooPlex: ~0.11 s in-cluster against 0.26–1.16 s out through the public // name, which adds a DNS lookup, the ingress hop and a TLS handshake to each one for nothing. // // Overridable so a hub running outside the cluster still works; the plain-HTTP default is // safe only because it never leaves the cluster network — do not point this at a public // host without https, the registry token rides the Authorization header. // // If the service is unreachable, artifactChoices logs a WARN and returns nil, and the form // degrades to manual text entry. That is the pre-existing behaviour and it stays. giteaAPI := os.Getenv("GITEA_API_URL") if giteaAPI == "" { giteaAPI = "http://gitea.gitea-system.svc.cluster.local:3000" } giteaClient = gitea.New(giteaAPI, "admin", cfg.Registry.Username, cfg.Registry.Token) webServer.SetGiteaClient(giteaClient) logger.Printf("[INFO] Gitea artifact browser enabled (Day-0 version dropdowns) via %s", giteaAPI) } // Offsite provisioning (SLICE 1): enabled when a Hetzner storage-box token is provided out-of-band. // PREREQUISITE: this MUST be a token scoped to a DEDICATED Hetzner project (the shared project token can // delete ep0 — SPIKE §6). Base is api.hetzner.com (NOT api.hetzner.cloud). Absent token → offsite UI // still renders, but saving with offsite enabled returns "not configured". var offsiteBoxChecker *monitor.OffsiteBoxChecker var pbsdrBoxChecker *monitor.PBSDRBoxChecker // R-5 v0.65.0: PBS-DR datastore fill (constructed with the tenantsync client below) // R-71c: the delivery checker's self-heal invokes the SAME Re-issue path the operator button // uses (webServer satisfies monitor.OffsiteReissuer). Wired ONLY when the provisioner exists — // a heal that cannot actually restage must never run (it would emit a restaged event for a // silent no-op: ReissueOffsiteForCustomer returns nil when offsite is unconfigured). var offsiteHealReissuer monitor.OffsiteReissuer var offsiteProv *offsite.Provisioner // R-32: the key service wires its purge into RESET's Deprovision below if tok := os.Getenv("HETZNER_TOKEN"); tok != "" { poolBoxID, _ := strconv.ParseInt(os.Getenv("HETZNER_POOL_BOX_ID"), 10, 64) location := os.Getenv("HETZNER_LOCATION") if location == "" { location = "fsn1" } client := hetznerapi.NewClient(func() string { return os.Getenv("HETZNER_TOKEN") }) offsiteProv = &offsite.Provisioner{ API: client, Store: dataStore, Scanner: offsite.SSHHostKeyScanner{}, PoolBoxID: poolBoxID, Location: location, Logger: logger, } webServer.SetOffsiteProvisioner(offsiteProv) logger.Printf("[INFO] Offsite provisioning enabled (pool_box=%d, location=%s)", poolBoxID, location) offsiteHealReissuer = webServer // R-71c heal armed (provisioner present) // R-5 (v0.64.0): the pool-box aggregate checker shares the SAME client + pool box id (GET-only). // It needs a valid box id to poll; without one, the aggregate stays unconfigured. if poolBoxID != 0 { offsiteBoxChecker = monitor.NewOffsiteBoxChecker(client, poolBoxID, dataStore, cfg.Alerting.OffsiteBoxFillWarnPercent, cfg.Alerting.OffsiteBoxFillCritPercent, cfg.Alerting.OffsiteOversubWarnRatio, cfg.Alerting.BoxUnreachableWindows, dispatcher.ProcessEvent, logger) webServer.SetOffsiteBox(offsiteBoxChecker.Snapshot) } else { logger.Printf("[INFO] Offsite pool-box aggregate disabled (HETZNER_POOL_BOX_ID unset)") } } // v0.57.0 (F3) — clean-slate re-enrollment auto re-issues the off-site login. Since hub v0.127.0 that // login is the hub's own (its key registrar); no box receives it (R-1158, `05` §8). // The API host-enroll path calls this seam; the web server owns the offsite provisioner + config // bump (and, via ReissueCredentials, the escrow invalidation + events). No-op when offsite is // unconfigured or the customer has no offsite tier. apiHandler.SetOffsiteReissuer(webServer.ReissueOffsiteForCustomer) // OS updates, guest fast lane (hub v0.130.0, `11` §8 step 2). A release is approved when every ring-0 box runs the // set, healthy, for OS_APPROVE_AFTER (default 24h) and through OS_APPROVE_NIGHTS night runs (default 1) — the // ruled "1–2 day wait". Either override is a TEST configuration and is logged loudly. osSvc := &osupdates.Service{Store: dataStore, Emit: dispatcher.ProcessEvent, Logger: logger, ApproveAfter: 24 * time.Hour, NightsRequired: 1, Bump: func(hostID string) { if _, err := dataStore.BumpHostDesired(hostID); err != nil { logger.Printf("[WARN] osupdates: bump desired for %s: %v", hostID, err) } }, // R-836 (`09` §3 decision 172): the household's day-before kernel mail; no accepted mail → no kernel step. KernelMail: dispatcher.SendKernelNotice} if v := os.Getenv("OS_APPROVE_AFTER"); v != "" { if d, derr := time.ParseDuration(v); derr == nil && d >= 0 { osSvc.ApproveAfter = d logger.Printf("[WARN] OS_APPROVE_AFTER=%s — OS releases approve after %s instead of 24h (TEST CONFIGURATION)", v, d) } else { logger.Printf("[ERROR] OS_APPROVE_AFTER=%q invalid — keeping 24h", v) } } if v := os.Getenv("OS_APPROVE_NIGHTS"); v != "" { if n, nerr := strconv.Atoi(v); nerr == nil && n >= 0 { osSvc.NightsRequired = n logger.Printf("[WARN] OS_APPROVE_NIGHTS=%s — OS releases need %d night run(s) instead of 1 (TEST CONFIGURATION)", v, n) } else { logger.Printf("[ERROR] OS_APPROVE_NIGHTS=%q invalid — keeping 1", v) } } osSvc.DockerNights = 2 if v := os.Getenv("OS_DOCKER_APPROVE_NIGHTS"); v != "" { if n, nerr := strconv.Atoi(v); nerr == nil && n >= 0 { osSvc.DockerNights = n if n == 0 { osSvc.DockerNights = -1 // "none" (0 in the service means the default) } logger.Printf("[WARN] OS_DOCKER_APPROVE_NIGHTS=%s — the Docker button needs %d healthy night(s) instead of 2 (TEST CONFIGURATION)", v, n) } else { logger.Printf("[ERROR] OS_DOCKER_APPROVE_NIGHTS=%q invalid — keeping 2", v) } } // `11` §5.3.1 (hub v0.133.0): an approval made under ANY of the three TEST overrides carries the test mark, and a start // without them cancels every test approval no real one superseded (no ring-1 box installs it from then on). var overrides []string for _, k := range []string{"OS_APPROVE_AFTER", "OS_APPROVE_NIGHTS", "OS_DOCKER_APPROVE_NIGHTS"} { if v := os.Getenv(k); v != "" { overrides = append(overrides, k+"="+v) } } osSvc.TestOverride = strings.Join(overrides, " ") if cancelled, cerr := osSvc.CancelTestReleases(); cerr != nil { logger.Printf("[ERROR] osupdates: cancelling test approvals at start: %v (cancelled so far: %v)", cerr, cancelled) } else if len(cancelled) > 0 { logger.Printf("[WARN] osupdates: %d TEST approval(s) cancelled at start: %s", len(cancelled), strings.Join(cancelled, ", ")) } logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired) logger.Printf("[INFO] osupdates: the Docker engine set is approved only by the operator, after %d healthy ring-0 night(s)", osSvc.DockerNightsEffective()) apiHandler.SetOSUpdateService(osSvc) webServer.SetOSUpdateAdmin(osSvc) // `11` §8.3: the four alarm thresholds are configuration (decided by CC unattended — operator may reverse). for _, a := range []struct { env string dst *time.Duration def time.Duration }{ {"OS_ALARM_STALE_AFTER", &osSvc.StaleAfter, 7 * 24 * time.Hour}, {"OS_ALARM_REBOOT_AFTER", &osSvc.RebootAfter, 14 * 24 * time.Hour}, {"OS_ALARM_RING0_STALL_AFTER", &osSvc.Ring0StallAfter, 7 * 24 * time.Hour}, {"OS_ALARM_NOT_COVERED_AFTER", &osSvc.NotCoveredAfter, 14 * 24 * time.Hour}, {"OS_ALARM_BUNDLE_BEHIND_AFTER", &osSvc.BundleBehindAfter, 7 * 24 * time.Hour}, {"OS_ALARM_AGENT_BEHIND_AFTER", &osSvc.AgentBehindAfter, 7 * 24 * time.Hour}, } { *a.dst = a.def if v := os.Getenv(a.env); v != "" { if d, derr := time.ParseDuration(v); derr == nil && d > 0 { *a.dst = d } else { logger.Printf("[ERROR] %s=%q invalid — keeping %s", a.env, v, a.def) } } } logger.Printf("[INFO] osupdates: alarms = no OS leg %s, reboot needed %s, ring 0 stalled %s, not covered %s", osSvc.StaleAfter, osSvc.RebootAfter, osSvc.Ring0StallAfter, osSvc.NotCoveredAfter) go func() { tk := time.NewTicker(60 * time.Second) defer tk.Stop() last := map[string]string{} var lastAlarms time.Time for { select { case <-ctx.Done(): return case <-tk.C: sts, err := osSvc.Evaluate() if err != nil { logger.Printf("[WARN] osupdates: evaluate: %v", err) } for _, st := range sts { if msg := st.Fingerprint + "|" + st.Waiting; msg != last[st.Layer] { last[st.Layer] = msg logger.Printf("[INFO] osupdates: %s candidate %s (%d packages, first seen %s): %s", st.Layer, st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""]) } } if time.Since(lastAlarms) >= time.Hour { lastAlarms = time.Now() start := time.Now() sent, aerr := osSvc.Alarms() if aerr != nil { logger.Printf("[WARN] osupdates: alarms: %v", aerr) } else { logger.Printf("[INFO] osupdates: alarm check done in %s, %d alarm(s) sent %v", time.Since(start).Round(time.Millisecond), len(sent), sent) } } } } }() // Decision 69 (v0.127.0): the off-site KEY REGISTRAR. The box sends its public key; the hub writes it // into the sub-account's authorized_keys pinned append-only; the daily check reads every file. if offsiteKeyReady { keySvc := &offsitekeys.Service{ Store: dataStore, Reg: &offsitekeys.Registrar{Dialer: offsitekeys.SSHDialer{}}, Logger: logger, Emit: dispatcher.ProcessEvent, } // Decision 74: the hub-enforced wait before a set-aside copy is deleted. OFFSITE_ABANDON_DELAY // overrides the 7-day default ONLY for a test, and is logged loudly when it does. keySvc.AbandonDelay = offsitekeys.DefaultAbandonDelay if v := os.Getenv("OFFSITE_ABANDON_DELAY"); v != "" { if d, derr := time.ParseDuration(v); derr == nil && d > 0 { keySvc.AbandonDelay = d logger.Printf("[WARN] OFFSITE_ABANDON_DELAY=%s — set-aside deletions wait %s instead of 7 days (TEST CONFIGURATION)", v, d) } else { logger.Printf("[ERROR] OFFSITE_ABANDON_DELAY=%q invalid — keeping 7 days", v) } } apiHandler.SetOffsiteKeyService(keySvc) if offsiteProv != nil { offsiteProv.PurgeShared = keySvc.PurgeRepos // R-32: RESET purges the folder through the sub-account's own login } runKeyAudit := func(ctx context.Context) any { start := time.Now() out := keySvc.AuditAll(ctx, dataStore.OffsiteWindowOpen) logger.Printf("[INFO] off-site key check: %d sub-account(s) read in %s", len(out), time.Since(start).Round(time.Millisecond)) type row struct { Customer string `json:"customer"` Lines int `json:"lines"` Pinned int `json:"pinned"` Findings []offsitekeys.Finding `json:"findings"` Error string `json:"error,omitempty"` } var rows []row for _, o := range out { rr := row{Customer: o.CustomerID, Lines: o.Result.Lines, Pinned: o.Result.Pinned, Findings: o.Result.Findings} if o.Err != nil { rr.Error = o.Err.Error() } rows = append(rows, rr) } return rows } webServer.SetOffsiteKeyAudit(runKeyAudit) webServer.SetOffsiteWindowAdmin(dataStore.GrantOffsiteWindowOnce, dataStore.SetOffsiteWindowsEnabled) webServer.SetOffsiteWindowLargeGrant(keySvc.GrantLargeWindow) webServer.SetOffsiteKeyAdmin(keySvc.RemoveUnpinnedKeys, keySvc.CancelAbandon) // Decision 68: a window the box never closed is closed by the hub at its 20-minute bound. go func() { tk := time.NewTicker(60 * time.Second) defer tk.Stop() for { select { case <-ctx.Done(): return case <-tk.C: sctx, cancel := context.WithTimeout(ctx, 2*time.Minute) keySvc.SweepExpiredWindows(sctx) keySvc.SweepAbandons(sctx) cancel() } } }() go scheduleDaily(ctx, "offsite-key-audit", "07:10", func() { actx, cancel := context.WithTimeout(ctx, 10*time.Minute) defer cancel() runKeyAudit(actx) }, logger) logger.Printf("[INFO] Off-site key registrar enabled; daily key check at 07:10 Budapest") } // Direction-2 immediate-sync (v0.58.0): one in-memory operator-intent notifier, shared by the // web handlers (which Bump it after every intent write) and the API handler (which long-polls it // at GET /api/v1/wait). In-memory BY DESIGN — a restart resets generations to zero; the box // compares with != , so a restart costs exactly one harmless full-state report. Closed before // server.Shutdown (below) so held waits complete instantly instead of eating the grace window. intentHub := intent.New() intentHub.SetLogger(logger.Printf) // R-30: one DEBUG line per box-presence change apiHandler.SetIntentHub(intentHub) webServer.SetIntentHub(intentHub) // Build HTTP mux mux := http.NewServeMux() // Health check endpoint — bypasses auth (for k8s probes) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { if err := dataStore.Ping(); err != nil { w.WriteHeader(http.StatusServiceUnavailable) w.Write([]byte("db unhealthy")) return } w.WriteHeader(http.StatusOK) w.Write([]byte("ok")) }) // API routes (no dashboard auth for report ingest) mux.Handle("/api/v1/", apiHandler) // Web routes — ALWAYS behind RequireAuth (v0.152.0). The old `if cfg.Auth.PasswordHash != ""` // mounted the UI with no auth at all when the ConfigMap seed was empty, even with a password set // in the DB override. With no password anywhere, RequireAuth now closes every operator route. // Pinned by TestWebMountAlwaysAuthenticated. if cfg.Auth.PasswordHash == "" { logger.Printf("[WARN] auth.password_hash is empty — the operator UI opens only if a password is set in the database; otherwise every operator route answers 503") } mux.Handle("/", webServer.Handler()) // R-1140 (operator decision A, 2026-10-17): the hub's OWN access log — one line per request, a file per UTC day under // /access-log on the hub's volume, kept 90 days. Path only (no query), no credential value; a logging // failure never fails or delays a request (internal/accesslog). Pinned by TestServerHandlerIsAccessLogged. accessLog := accesslog.New(accesslog.Options{Dir: filepath.Join(cfg.Server.DataDir, "access-log"), Logf: logger.Printf}) logger.Printf("[INFO] access log: %s, one file per UTC day, kept %d days", filepath.Join(cfg.Server.DataDir, "access-log"), int(accesslog.DefaultKeep.Hours()/24)) // Start HTTP server server := &http.Server{ Addr: cfg.Server.Listen, Handler: accessLog.Middleware(mux), ReadTimeout: 30 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 120 * time.Second, } // Initialize version checker for controller image registry var versionChecker *web.VersionChecker if cfg.Registry.Username != "" && cfg.Registry.Token != "" { checkInterval, err := time.ParseDuration(cfg.Registry.CheckInterval) if err != nil { checkInterval = 6 * time.Hour } versionChecker = web.NewVersionChecker(cfg.Registry.Image, cfg.Registry.Username, cfg.Registry.Token, checkInterval, logger) go versionChecker.Run(ctx) logger.Printf("[INFO] Registry version checker started (every %s)", cfg.Registry.CheckInterval) } else { logger.Printf("[INFO] Registry version checker disabled (no credentials configured)") } webServer.SetVersionChecker(versionChecker) // Advertise latest on the controller report ACK (Phase 2). Guard the nil case so we don't store a // non-nil interface wrapping a nil *VersionChecker. if versionChecker != nil { apiHandler.SetLatestVersionProvider(versionChecker) } // S1 offsite connectivity: the WG peer-sync reconciler (internal/wgsync). Config from env // (mirror of the Resend/registry pattern): the SSH private key from the mounted Secret file, // the (non-secret) pinned host key from plain env. Any piece missing → disabled with an INFO // log; the /admin/wg mutations still work against the DB and report sync:"disabled". { wgAddr := os.Getenv("WG_ENDPOINT_SSH_ADDR") wgUser := os.Getenv("WG_ENDPOINT_SSH_USER") if wgUser == "" { wgUser = "felhom-peersync" } wgKeyFile := os.Getenv("WG_ENDPOINT_SSH_KEY_FILE") wgHostKey := os.Getenv("WG_ENDPOINT_SSH_HOSTKEY") if wgAddr != "" && wgKeyFile != "" && wgHostKey != "" { keyPEM, err := os.ReadFile(wgKeyFile) if err != nil { logger.Printf("[ERROR] WG peer-sync disabled: read key file %s: %v", wgKeyFile, err) } else if wgClient, err := wgsync.New(wgsync.Config{ Addr: wgAddr, User: wgUser, PrivateKey: keyPEM, HostKeyLine: wgHostKey, }, logger); err != nil { logger.Printf("[ERROR] WG peer-sync disabled: %v", err) } else { wgReconciler := wgsync.NewReconciler(dataStore, wgClient, logger) go wgReconciler.Run(ctx) apiHandler.SetWGSyncer(wgReconciler) webServer.SetWGPeerSync(wgReconciler.Trigger) // R-600: the customer delete pushes at once logger.Printf("[INFO] WG peer-sync enabled (endpoint %s, user %s)", wgAddr, wgUser) } } else { logger.Printf("[INFO] WG peer-sync disabled (endpoint not configured)") } // PBS DR tier (SLICE 1): the tenantsync client — same endpoint address + pinned host key // as peersync, its OWN private key (the authorized_keys line selects the forced command). tsKeyFile := os.Getenv("TENANTSYNC_SSH_KEY_FILE") if wgAddr != "" && wgHostKey != "" && tsKeyFile != "" { keyPEM, err := os.ReadFile(tsKeyFile) if os.IsNotExist(err) { // The env names the mount path unconditionally; the optional Secret may not exist yet. logger.Printf("[INFO] PBS DR tenantsync disabled (key %s not present)", tsKeyFile) } else if err != nil { logger.Printf("[ERROR] PBS DR tenantsync disabled: read key file %s: %v", tsKeyFile, err) } else if tsClient, err := tenantsync.New(tenantsync.Config{ Addr: wgAddr, User: wgUser, PrivateKey: keyPEM, HostKeyLine: wgHostKey, }, logger); err != nil { logger.Printf("[ERROR] PBS DR tenantsync disabled: %v", err) } else { webServer.SetTenantSync(tsClient) // v0.51.0 DR-tier cascade (scenario A): a host's first WG registration may be // the descriptor's last unmet precondition — auto-provision hands-free. apiHandler.SetWGRegisteredHook(webServer.PBSDRAutoProvision) logger.Printf("[INFO] PBS DR tenantsync enabled (endpoint %s, user %s; WG-registration auto-provision hook armed)", wgAddr, wgUser) // R-5 (v0.65.0): the PBS-DR datastore fill checker shares the SAME tenantsync client // (read-only usage op). Graceful vs an ep0 still on script ≤ v1.1.0 (unavailable state). pbsdrBoxChecker = monitor.NewPBSDRBoxChecker(tsClient, cfg.Alerting.PBSDRBoxFillWarnPercent, cfg.Alerting.PBSDRBoxFillCritPercent, cfg.Alerting.BoxUnreachableWindows, dispatcher.ProcessEvent, logger) webServer.SetPBSDRBox(pbsdrBoxChecker.Snapshot) } } else { logger.Printf("[INFO] PBS DR tenantsync disabled (key or endpoint not configured)") } // Agent-plane immediate-sync SENDER (Direction-2a, v0.59.0; SPIKE-immediate-sync-transport): // same endpoint + pinned host key + peersync user as wgsync/tenantsync, its OWN forced-command // key (POKE_SSH_KEY_FILE, optional Secret). A poke is contentless + fire-and-forget; if this // is unconfigured, an agent-plane save just bumps the generation and the box picks it up on its // next ≤15-min report (no error to the operator). pokeKeyFile := os.Getenv("POKE_SSH_KEY_FILE") if wgAddr != "" && wgHostKey != "" && pokeKeyFile != "" { keyPEM, err := os.ReadFile(pokeKeyFile) if os.IsNotExist(err) { logger.Printf("[INFO] agent-plane poke disabled (key %s not present)", pokeKeyFile) } else if err != nil { logger.Printf("[ERROR] agent-plane poke disabled: read key file %s: %v", pokeKeyFile, err) } else if pokeClient, err := poke.New(poke.Config{ Addr: wgAddr, User: wgUser, PrivateKey: keyPEM, HostKeyLine: wgHostKey, }, logger); err != nil { logger.Printf("[ERROR] agent-plane poke disabled: %v", err) } else { // One notifier, both planes' system-initiated sites (v0.63.0): the web server's // pbsdr writes AND the api handler's admin desired-state / operator-peer writes. n := poke.NewNotifier(dataStore, pokeClient, logger) webServer.SetPoke(n) apiHandler.SetPoker(n) logger.Printf("[INFO] agent-plane poke enabled (endpoint %s, user %s; web + api admin seams armed)", wgAddr, wgUser) } } else { logger.Printf("[INFO] agent-plane poke disabled (key or endpoint not configured)") } } // PBS-DR self-heal reconciler (internal/pbsdrheal, from SPIKE-pbsdr-selfheal-2026-07-15). Re-arms // a consumable secret for boxes stuck in waiting_secret/consumed_failed after losing their // converged marker (re-install / restore / snapshot rollback onto a stable host_id — the case no // existing path recovers). The primary heal (re-stage the stored secret) needs no endpoint; the // escalation reuses webServer.ReissuePBSDR (a no-op-error if tenantsync is unconfigured). Reads // the hub DB only; a converged/disabled host is a pure no-op. PBSDRHEAL_ONLY_HOST scopes a // supervised first rollout to one host (empty = whole fleet — the steady state). { pbsdrReconciler := pbsdrheal.NewReconciler(dataStore, pbsdrheal.NewActions(dataStore, webServer), logger) if only := os.Getenv("PBSDRHEAL_ONLY_HOST"); only != "" { pbsdrReconciler.RestrictToHost(only) logger.Printf("[INFO] PBS-DR self-heal reconciler RESTRICTED to host %s (supervised rollout scope)", only) } go pbsdrReconciler.Run(ctx) logger.Printf("[INFO] PBS-DR self-heal reconciler started (interval 5m)") } // Off-site credential self-heal reconciler (internal/offsiteheal, R-193 / R-204 item 4). The // sibling of the block above, for the last of the four manual interventions the 2026-08-04 drill // needed: a REBUILT box has no off-site credential because its predecessor spent the one-time // password. It acts ONLY on a state the box DECLARES (an absence has four meanings and the hub // cannot tell them apart; the box can), sustained across two distinct reports, re-staging the // stored secret before ever minting. A healthy box is a pure no-op. OFFSITEHEAL_ONLY_CUSTOMER // scopes a supervised first rollout (empty = whole fleet — the steady state). { offsiteReconciler := offsiteheal.NewReconciler(dataStore, offsiteheal.NewActions(dataStore, webServer), logger) if only := os.Getenv("OFFSITEHEAL_ONLY_CUSTOMER"); only != "" { offsiteReconciler.RestrictToCustomer(only) logger.Printf("[INFO] offsite credential self-heal RESTRICTED to customer %s (supervised rollout scope)", only) } go offsiteReconciler.Run(ctx) logger.Printf("[INFO] offsite credential self-heal reconciler started (interval 5m, debounce 2 reports)") } // Session cleanup — removes expired sessions every hour go webServer.CleanupSessions(ctx) // Prune on startup, then daily at configured time (default 04:30) if cfg.Retention.MaxDays > 0 { pruneAll(dataStore, cfg.Retention.MaxDays, logger) go scheduleDaily(ctx, "prune", cfg.Retention.PruneSchedule, func() { pruneAll(dataStore, cfg.Retention.MaxDays, logger) }, logger) } // Staleness checker — runs every 60s stalenessChecker := monitor.NewStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger) // v0.7.0: host-domain dead-man's-switch (sibling; the controller checker above is // unchanged and keeps running until the slice-10 cutover). Same 60s cadence. hostStalenessChecker := monitor.NewHostStalenessChecker(dataStore, staleThreshold, dispatcher.ProcessEvent, logger) // v0.44.0-agent: operator alert when an agent reports a degraded privileged capability (a missing // `sudo -n` grant — the 2026-06-28 cutover class). Same 60s sweep + transition/cooldown plumbing. hostCapabilityChecker := monitor.NewHostCapabilityChecker(dataStore, dispatcher.ProcessEvent, logger) // agent-v0.48.0: proactive fleet-wide agent-re-key detection — alert when a host's reported // local-API leaf fingerprint changes (independent of the controller channel-check). Same 60s sweep. hostLeafChecker := monitor.NewHostLeafChecker(dataStore, dispatcher.ProcessEvent, logger) // v0.23.0: HOST root-fs disk-pressure alert (the silent vzdump-on-root-fills class). Reads the host // root disk_percent the agent reports; born/persistent (a disk already full at hub restart alerts on // cycle 1). Distinct event types from the controller's GUEST disk_warning/disk_critical. Same 60s sweep. hostDiskChecker := monitor.NewHostDiskChecker(dataStore, cfg.Alerting.HostDiskWarnPercent, cfg.Alerting.HostDiskCritPercent, dispatcher.ProcessEvent, logger) // v0.25.0: PER-STORAGE worst-fill alert — generalizes the host-root check to any reported storage // target (dump/backup volume, data drive, lvmthin pool, PBS datastore). Born/persistent; excludes the // root-backed builtin (hostDiskChecker owns root, no double-alert); emits natural `critical`. Same sweep. storageFillChecker := monitor.NewStorageFillChecker(dataStore, cfg.Alerting.StorageFillWarnPercent, cfg.Alerting.StorageFillCritPercent, dispatcher.ProcessEvent, logger) // TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep // dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep. hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger) // TASK H1: alert when a host's OPERATOR ACCESS is degraded — felhom-sshd down (with the operator // peer configured) or its config invalid. Transition-based, same 60s sweep. hostOOBChecker := monitor.NewHostOOBChecker(dataStore, dispatcher.ProcessEvent, logger) // SLICE 4: offsite backup health from the controller report's `offsite` object — soft-quota fill // (90/95% of quota_gb) + staleness (enabled+escrowed but no run >48h — the silently-stuck detector; // run FAILURES already alert via backup_failed). Nil-safe on pre-v0.109 reports. Same sweep. offsiteChecker := monitor.NewOffsiteChecker(dataStore, 0, dispatcher.ProcessEvent, logger) // R-85: the restore-test result becomes a signal instead of a log line. Two distinct events // (failure vs staleness), operator-tier only — neither has a customerMessages entry. restoreTestChecker := monitor.NewRestoreTestChecker(dataStore, dispatcher.ProcessEvent, logger) // R-70 + R-71c: the delivery-state checker — surfaces the burned-credential shape as // offsite_delivery_stuck (warning, 24h/customer) and self-heals it via the Re-issue path // (offsite_credential_restaged, one restage/customer/24h, R-39(a)-guarded). Cooldowns are // durable (events table), so a hub restart neither floods nor silently re-heals. // R-523: the agent's in-guest controller supervisor → controller_restarted_by_agent / controller_crashloop. controllerSupervisorChecker := monitor.NewControllerSupervisorChecker(dataStore, dispatcher.ProcessEvent, logger) offsiteDeliveryChecker := monitor.NewOffsiteDeliveryChecker(dataStore, offsiteHealReissuer, dispatcher.ProcessEvent, logger) // R-242 second half (S4, decision 199): published golden vs vouched golden. Needs the registry creds // (the same Gitea client as the Day-0 dropdowns); without them the check is off and says so. var goldenVouchChecker *monitor.GoldenVouchChecker if giteaClient != nil { goldenVouchChecker = monitor.NewGoldenVouchChecker( func(c context.Context) ([]string, error) { return giteaClient.ListVersions(c, "felhom-golden") }, func() string { return dataStore.GetArtifactManifest().GoldenVersion }, dispatcher.ProcessEvent, logger) } else { logger.Printf("[WARN] Golden vouch check OFF: no registry credentials, so the hub cannot list published goldens") } go func() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: stalenessChecker.Check() hostStalenessChecker.Check() hostCapabilityChecker.Check() hostLeafChecker.Check() hostDiskChecker.Check() storageFillChecker.Check() hostMgmtPlaneChecker.Check() hostOOBChecker.Check() offsiteChecker.Check() restoreTestChecker.Check() // R-85: restore-test failure + per-tier staleness offsiteDeliveryChecker.Check() controllerSupervisorChecker.Check() if goldenVouchChecker != nil { goldenVouchChecker.Check() // R-242: throttled to one Gitea listing per 15 min } if offsiteBoxChecker != nil { offsiteBoxChecker.Check() // R-5: restic pool-box aggregate (fetch-throttled internally) } if pbsdrBoxChecker != nil { pbsdrBoxChecker.Check() // R-5 v0.65.0: PBS-DR datastore fill (ep0 usage op, throttled) } // v0.46.0: pulled log bundles are transient diagnostics — 72 h TTL. if n, perr := dataStore.PurgeExpiredLogBundles(time.Now()); perr != nil { logger.Printf("[WARN] log-bundle TTL purge failed: %v", perr) } else if n > 0 { logger.Printf("[INFO] log-bundle TTL purge: %d expired bundle(s) dropped", n) } } } }() // DB snapshot every 6 hours (R-173, v0.136.0, `05` §16.3; D34 v0.167.0): VACUUM INTO /snapshots/, keep 2. // DooPlex's felhom-hub-db-backup.service picks up the newest at :30 after each slot, checks, encrypts and pushes it // to ep0. A start-up catch-up runs one when the newest is older than one slot gap (a pod that was down at a slot). snapMaker := &dbsnap.Maker{Store: dataStore, Dir: filepath.Join(cfg.Server.DataDir, "snapshots"), Logger: logger} runSnapshot := func() { sctx, cancel := context.WithTimeout(ctx, 10*time.Minute) defer cancel() if _, err := snapMaker.Make(sctx); err != nil { logger.Printf("[ERROR] db snapshot failed: %v", err) } } if dbsnap.NeedsCatchUp(snapMaker.Dir, time.Now(), dbsnap.CatchUpAge) { go runSnapshot() } for _, at := range dbsnap.Slots { go scheduleDaily(ctx, "db-snapshot "+at, at, runSnapshot, logger) } // Backup deadline checker — runs daily at 05:00 Budapest go scheduleDaily(ctx, "deadline-check", "05:00", func() { monitor.CheckBackupDeadlines(dataStore, stalenessChecker, dispatcher.ProcessEvent, logger) }, logger) // Signal handling sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) go func() { sig := <-sigCh logger.Printf("[INFO] Received signal %v, shutting down...", sig) cancel() // Complete every held long-poll before Shutdown so they don't consume the grace window. intentHub.Close() shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second) defer shutdownCancel() if err := server.Shutdown(shutdownCtx); err != nil { logger.Printf("[ERROR] HTTP server shutdown error: %v", err) } }() logger.Printf("[INFO] Listening on %s", cfg.Server.Listen) if err := server.ListenAndServe(); err != http.ErrServerClosed { logger.Fatalf("[FATAL] HTTP server error: %v", err) } accessLog.Close() // drains the buffered lines of the last requests logger.Println("[INFO] felhom-hub stopped") } func loadConfig(path string, logger *log.Logger) *Config { cfg := &Config{} // Defaults cfg.Server.Listen = ":8080" cfg.Server.DataDir = "/data" cfg.Retention.MaxDays = 90 cfg.Retention.PruneSchedule = "04:30" cfg.Alerting.StaleThreshold = "30m" data, err := os.ReadFile(path) if err != nil { logger.Printf("[WARN] Config file not found at %s, using defaults", path) return cfg } if err := yaml.Unmarshal(data, cfg); err != nil { logger.Printf("[WARN] Failed to parse config: %v, using defaults", err) return cfg } // Apply defaults for zero values if cfg.Server.Listen == "" { cfg.Server.Listen = ":8080" } if cfg.Server.DataDir == "" { cfg.Server.DataDir = "/data" } if cfg.Retention.MaxDays == 0 { cfg.Retention.MaxDays = 90 } if cfg.Alerting.StaleThreshold == "" { cfg.Alerting.StaleThreshold = "30m" } if cfg.Notifications.FromEmail == "" { cfg.Notifications.FromEmail = "monitoring@felhom.eu" } if cfg.Registry.Image == "" { cfg.Registry.Image = "gitea.dooplex.hu/admin/felhom-controller" } if cfg.Registry.CheckInterval == "" { cfg.Registry.CheckInterval = "6h" } if cfg.Registry.TemplateInterval == "" { cfg.Registry.TemplateInterval = "1h" } return cfg } // scheduleDaily runs fn once daily at the given "HH:MM" time in Europe/Budapest. // It blocks until ctx is cancelled. func scheduleDaily(ctx context.Context, name, timeStr string, fn func(), logger *log.Logger) { budapest, err := time.LoadLocation("Europe/Budapest") if err != nil { budapest = time.FixedZone("CET", 3600) } hour, min := parseHM(timeStr) var last time.Time for { next := nextDailyRun(time.Now(), last, hour, min, budapest) delay := time.Until(next) logger.Printf("[INFO] %s: next run at %s (in %s)", name, next.Format("2006-01-02 15:04 MST"), delay.Round(time.Second)) select { case <-ctx.Done(): return case <-time.After(delay): last = next fn() } } } // nextDailyRun returns the next HH:MM in loc that is after now and after the slot that ran last (the zero time // when none has). R-1183: the day after is computed as a calendar day (time.Date with day+1), never as now+24h — // on 2026-10-25 (25 hours) the old Add(24h) fired a 04:00 job at 03:00 CET and again at 04:00, and on 2027-03-28 // (23 hours) it fired at 05:00 CEST. The controller's scheduler made the same fix (scheduler.go, day+1). // The `last` guard keeps a timer that wakes a moment early from running the same slot twice. // Pinned by TestR1183_* in r1183_dst_schedule_test.go. func nextDailyRun(now, last time.Time, hour, min int, loc *time.Location) time.Time { if last.After(now) { now = last } now = now.In(loc) for d := 0; ; d++ { next := time.Date(now.Year(), now.Month(), now.Day()+d, hour, min, 0, 0, loc) if next.After(now) { return next } } } // parseHM parses "HH:MM" into hour and minute. Returns 0, 0 on invalid input. func parseHM(s string) (int, int) { var h, m int if _, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil { return 0, 0 } return h, m } func pruneAll(s *store.Store, maxDays int, logger *log.Logger) { if deleted, err := s.Prune(maxDays); err != nil { logger.Printf("[WARN] Prune reports failed: %v", err) } else if deleted > 0 { logger.Printf("[INFO] Pruned %d old report rows", deleted) } if deleted, err := s.PruneEvents(maxDays); err != nil { logger.Printf("[WARN] Prune events failed: %v", err) } else if deleted > 0 { logger.Printf("[INFO] Pruned %d old event rows", deleted) } if n, err := s.PruneAppTelemetry(time.Now().Add(-90 * 24 * time.Hour)); err != nil { logger.Printf("[ERROR] Prune app telemetry: %v", err) } else if n > 0 { logger.Printf("[INFO] Pruned %d old app telemetry rows", n) } if n, err := s.PruneStaleIssues(time.Now().Add(-30 * 24 * time.Hour)); err != nil { logger.Printf("[ERROR] Prune stale issues: %v", err) } else if n > 0 { logger.Printf("[INFO] Pruned %d stale app issues", n) } // R-901 (`09` §3 decision 181): a deleted customer's audit rows go 1 year after the deletion. if ev, nl, err := s.PruneDeletedCustomerAudit(time.Now().UTC(), store.DeletedCustomerAuditKeep); err != nil { logger.Printf("[ERROR] Prune deleted customers' audit rows: %v", err) } else if ev+nl > 0 { logger.Printf("[INFO] Pruned a deleted customer's audit rows (older than 1 year after the deletion): events=%d notification_log=%d", ev, nl) } } // runUnsealBoxSecrets is `felhom-hub -unseal-box-secrets` (R-879 roll-back). It installs the sealing key // exactly as the server does, opens every sealed box secret back to plaintext (all or nothing), logs // counts only, and returns the process exit code. It never starts the server and never seals anything. func runUnsealBoxSecrets(st *store.Store, logger *log.Logger) int { key, err := store.ParseOffsiteSecretKey(os.Getenv("OFFSITE_SECRET_KEY")) if err != nil { logger.Printf("[ERROR] unseal-box-secrets: OFFSITE_SECRET_KEY: %v — nothing changed", err) return 2 } if err := st.SetOffsiteSecretKey(key); err != nil { logger.Printf("[ERROR] unseal-box-secrets: %v — nothing changed", err) return 2 } n, err := st.UnsealBoxSecrets() if err != nil { logger.Printf("[ERROR] unseal-box-secrets: %v", err) return 1 } logger.Printf("[INFO] unseal-box-secrets: %d sealed value(s) opened back to plaintext; a hub <= 0.137.0 can run on this database now", n) return 0 }