Files
felhom-controller/controller/cmd/controller/main.go
T
admin 86ea482fc1 controller v0.178.0 — R-88 Part 2: only a positive 'never' fires the valve
MinAgent: 0.105.0. scheduledRunAllowed fired on any nil age; it now requires a
licence from valveLicensed, which grants it for AgeStateAbsent and for a LEGACY
agent, and refuses it for AgeStateUnknown. An unreadable storage no longer
masquerades as a first-ever backup and no longer quiesces apps outside the window.

A missing wire field means legacy, not unknown — deliberately. Treating it as
unknown would stop the valve firing on un-upgraded boxes and starve genuinely new
ones. Degrade logged once; unrecognised future values also map to legacy.

Caught in passing: TieredBackend is satisfied by a RUNTIME assertion, so the
signature change compiled and vetted clean while quiesceBackend silently stopped
satisfying it — which would have degraded every box to the single-tier path with
no error. Added a compile-time witness.

Also corrects the notifier comment that claimed operator-only came from a missing
customerMessages entry; enforcement is hub-side operatorOnlyEvents (hub 0.79.0).
2026-07-27 18:08:56 +02:00

1967 lines
84 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"syscall"
"time"
"crypto/subtle"
"strings"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
"gitea.dooplex.hu/admin/felhom-controller/internal/api"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
"gitea.dooplex.hu/admin/felhom-controller/internal/assets"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
"gitea.dooplex.hu/admin/felhom-controller/internal/mailrelay"
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
"gitea.dooplex.hu/admin/felhom-controller/internal/offsiteapply"
"gitea.dooplex.hu/admin/felhom-controller/internal/quiesce"
"gitea.dooplex.hu/admin/felhom-controller/internal/recovery"
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
"gitea.dooplex.hu/admin/felhom-controller/internal/selftest"
"gitea.dooplex.hu/admin/felhom-controller/internal/selfupdate"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/setup"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
catalogsync "gitea.dooplex.hu/admin/felhom-controller/internal/sync"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
"gitea.dooplex.hu/admin/felhom-controller/internal/web"
)
var (
// Set at build time via ldflags
Version = "dev"
BuildTime = "unknown"
GitCommit = "unknown"
)
func main() {
// Hidden re-exec mode (NAS verify-before-commit): the web server re-execs THIS binary as
// `felhom-controller --netprobe <dir>` with uid/gid-1000 credentials to prove a media app can
// write through a freshly-verified network share (SPIKE-nas-verify Q2). Handled before flag
// parsing; the exit code is the probe verdict (see web/netprobe.go).
if len(os.Args) == 3 && os.Args[1] == "--netprobe" {
os.Exit(web.NetProbeChild(os.Args[2]))
}
configPath := flag.String("config", "/opt/docker/felhom-controller/controller.yaml", "Path to configuration file")
showVersion := flag.Bool("version", false, "Show version and exit")
printResetCode := flag.Bool("print-reset-code", false, "Customer-claim escape hatch (v0.122.0, F-4): print a fresh one-time local claim/reset code to stdout, then exit. Root-gated by reachability (docker exec). Same gate consumes it.")
printInfraImages := flag.Bool("print-infra-images", false, "Print every controller-managed infra image (one per line) and exit. Read by the golden bake (felhom-agent configs/build-golden.sh) so the appliance image pre-pulls exactly what THIS controller version will request.")
flag.Parse()
if *showVersion {
fmt.Printf("felhom-controller %s (built %s, commit %s)\n", Version, BuildTime, GitCommit)
os.Exit(0)
}
// Config-free by design: the golden bake runs this against a bare `docker run` of the controller
// image, where no controller.yaml, data dir or settings exist yet. It must never touch either.
if *printInfraImages {
for _, img := range infra.Images() {
fmt.Println(img)
}
os.Exit(0)
}
if *printResetCode {
cfg, err := config.LoadPermissive(*configPath)
if err != nil {
fmt.Fprintf(os.Stderr, "print-reset-code: loading config: %v\n", err)
os.Exit(1)
}
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", log.New(os.Stderr, "", 0))
if err != nil {
fmt.Fprintf(os.Stderr, "print-reset-code: loading settings: %v\n", err)
os.Exit(1)
}
os.Exit(web.PrintLocalResetCode(sett, cfg))
}
startTime := time.Now()
// --- Load configuration ---
// Use LoadPermissive to tolerate minimal configs (e.g. only domain set by docker-setup.sh).
// If even that fails (file missing/unreadable), fall back to defaults.
cfg, err := config.LoadPermissive(*configPath)
if err != nil {
cfg = config.Default()
log.Printf("[WARN] Config load failed (%s), using defaults: %v", *configPath, err)
}
logger, logBuffer := setupLogger(cfg)
// fix-6 (CAMPAIGN-3): load the pre-restart debug-ring window back immediately, so a controller
// restart / container recreation no longer wipes the exact evidence an operator needs. The spill
// lives on the SSD state dir (DataDir — holds settings.json, survives recreation), NEVER a NAS path.
ringSpillPath := filepath.Join(cfg.Paths.DataDir, "debug-ring.log")
logBuffer.LoadFrom(ringSpillPath)
// v0.116.0: the debug ring is the controller_log_tail source (report self-tail channel).
report.SetControllerLogSource(logBuffer.Lines)
// --- Bootstrap ingestion (slice 8A → v0.40.0 onboarding, doc 03 §6) ---
// On first run, if this controller is not yet configured AND the host agent's provisioning
// back-half attached a bootstrap.json config mount, PULL the full controller.yaml from the hub
// (using the bootstrap's retrieval passphrase), merge in the per-guest local_api block, and come
// up CONFIGURED — skipping setup mode. Idempotent (never clobbers an existing controller.yaml)
// and fail-safe (a malformed/absent bootstrap, or a hub outage at first boot, leaves us in setup
// mode). The adapter marks a transient hub-unreachable error as retryable (the rest are permanent).
pull := func(hubURL, customerID, retrievalPassword string) (string, error) {
y, perr := report.PullConfig(hubURL, customerID, retrievalPassword)
if perr != nil && errors.Is(perr, report.ErrHubUnreachable) {
return "", fmt.Errorf("%w: %w", bootstrap.ErrPullTransient, perr)
}
return y, perr
}
cfg = bootstrap.MaybeIngest(*configPath, cfg, logger, pull)
// --- Wire system package debug logging ---
if cfg.Logging.Level == "debug" {
system.DebugLogger = logger
}
// --- Setup mode: if no customer ID configured, run setup wizard ---
if setup.NeedsSetup(cfg) {
logger.Printf("[INFO] felhom-controller %s — setup mode", Version)
runSetupMode(cfg, logger)
return
}
// --- Local API connectivity probe (slice 8A) ---
// When seeded with a local-API endpoint, prove the controller↔agent channel at startup and
// learn this guest's mounts (placement view). Non-fatal — the controller runs regardless; a
// failure is logged for diagnosis. The full /backup/due quiesce loop lands in 8B.
probeLocalAPI(cfg, logger)
logger.Printf("[INFO] felhom-controller %s starting (customer: %s, domain: %s)",
Version, cfg.Customer.ID, cfg.Customer.Domain)
// --- Load settings ---
settingsPath := cfg.Paths.DataDir + "/settings.json"
sett, err := settings.Load(settingsPath, logger)
if err != nil {
logger.Fatalf("[FATAL] Failed to load settings from %s: %v", settingsPath, err)
}
sett.SetDebug(cfg.Logging.Level == "debug")
// --- Auto-discover storage paths from deployed apps ---
discoveredPaths := discoverHDDPaths(cfg.Paths.StacksDir, logger)
sett.AutoDiscoverStoragePaths(discoveredPaths, cfg.Paths.HDDPath, logger)
// --- Load or create encryption key ---
encKeyPath := filepath.Join(cfg.Paths.DataDir, "encryption.key")
encKey, err := crypto.LoadOrCreateKey(encKeyPath)
if err != nil {
logger.Fatalf("[FATAL] Failed to load encryption key: %v", err)
}
logger.Printf("[INFO] Encryption key loaded from %s", encKeyPath)
// --- Initialize stack manager ---
stackMgr, err := stacks.NewManager(cfg, logger)
if err != nil {
logger.Fatalf("[FATAL] Failed to initialize stack manager: %v", err)
}
stackMgr.SetEncryptionKey(encKey)
// Initial stack scan
if err := stackMgr.ScanStacks(); err != nil {
logger.Printf("[WARN] Initial stack scan failed: %v", err)
}
// Inject missing deploy fields for all deployed stacks on startup
if names := stackMgr.DeployedStackNames(); len(names) > 0 {
stackMgr.InjectMissingFields(names)
}
// Migrate existing plaintext passwords to encrypted
stackMgr.MigrateEncryption()
// --- First-boot base-infrastructure bring-up ---
// We are guaranteed configured here (setup.NeedsSetup returned false above), so deploy the base
// stack (traefik-public network → traefik → cloudflared → filebrowser) the controller needs for
// routing + external access. Runs in a goroutine so a slow first-boot image pull never delays the
// web server; non-fatal (idempotent + single-flight, the health loop re-attempts each tick).
go func() {
if err := stackMgr.EnsureBaseStack(); err != nil {
logger.Printf("[WARN] [infra] first-boot base-stack bring-up: %v", err)
}
}()
// --- Initialize catalog syncer ---
syncer := catalogsync.New(cfg, logger, stackMgr.ScanStacks, func(updated []string) {
stackMgr.InjectMissingFields(updated)
})
syncer.Start()
defer syncer.Stop()
// --- Graceful shutdown context ---
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// --- Quiesce loop (slice 8B): app-consistent backup around the agent vzdump ---
// Runs only when the local API is configured (a provisioned guest) and quiesce is enabled.
// Recover FIRST (restart any stacks left stopped by a crash mid-quiesce), then start the loop.
quiesceLoop := startQuiesceLoop(ctx, cfg, sett, stackMgr, logger)
// --- R-52: boot desired-state reconciliation ---
// A deployed app that missed its boot start used to stay down until a human noticed (F5: immich
// and calibre-web sat Exited for ~18 h while ten siblings came back). One bounded start-once
// sweep, deliberately AFTER the quiesce recovery above so the two never race for the same stack,
// and entirely inside deadAppBootGrace so a successful recovery is silent and a failed one still
// alerts honestly. Never touches an app the customer stopped — see internal/bootrecon.
go runBootReconcile(ctx, stackMgr, logger)
// --- Start CPU collector ---
cpuCollector := system.NewCPUCollector(5 * time.Second)
cpuCollector.Start(ctx)
defer cpuCollector.Stop()
// --- Initialize metrics store + collector ---
metricsDBPath := "/opt/docker/felhom-controller/data/metrics.db"
metricsStore, err := metrics.NewMetricsStore(metricsDBPath, logger)
if err != nil {
logger.Printf("[WARN] Failed to initialize metrics store: %v — monitoring disabled", err)
} else {
logger.Printf("[INFO] Metrics store opened at %s", metricsDBPath)
}
if metricsStore != nil {
defer metricsStore.Close()
metricsHDDPath := cfg.Paths.HDDPath
if p := sett.GetDefaultStoragePath(); p != "" {
metricsHDDPath = p
}
metricsCollector := metrics.NewMetricsCollector(metricsStore, cpuCollector, metricsHDDPath, logger)
metricsCollector.Start(ctx)
defer metricsCollector.Stop()
logger.Println("[INFO] Metrics collector started (60s interval)")
}
// Deprecation notice for ping UUIDs (Healthchecks pinging retired — the Hub
// now owns monitoring; disk-tier backup moved to the host agent in slice 8C).
uuids := cfg.Monitoring.PingUUIDs
if uuids.Heartbeat != "" || uuids.SystemHealth != "" || uuids.DBDump != "" || uuids.Backup != "" || uuids.BackupIntegrity != "" {
logger.Println("[INFO] Healthchecks ping UUIDs configured but no longer used — monitoring is now handled by the Hub")
}
// --- Initialize backup manager (app-data only: DB dumps + Docker-volume tars) ---
var backupMgr *backup.Manager
stackProv := &stackAdapter{
mgr: stackMgr,
getStoragePaths: func() []settings.StoragePath { return sett.GetStoragePaths() },
encKey: encKey,
}
if cfg.Backup.Enabled {
backupMgr = backup.NewManager(cfg, sett, logger)
backupMgr.SetStackProvider(stackProv)
backupMgr.SetVersion(Version)
// O4: restore-from-unit generates a replacement for an unrecoverable RESETTABLE secret
// (data-keys stay fail-closed) so the app redeploys with a fresh credential, not a blank one.
backupMgr.SetSecretGenerator(stackMgr.GenerateSecretForField)
// R-7b: after a shares restore re-adds definitions to the registry, smb.conf must be
// re-rendered or the restored shares exist on paper but are not exported. A seam rather than a
// direct call — the backup package must not depend on the stacks package.
backupMgr.SetSharesReconciler(stackMgr.ReconcileSamba)
}
// SLICE 2: the offsite apply-bridge is launched further down, AFTER the self-updater is constructed
// (R-71a: the bridge's settle-gate reads the updater's floor/update-running state to defer the
// consume past a managed day-0 floor-update). See "offsite apply-bridge" below.
// --- Wire the data-migration engine (B1) + backup↔migration mutual exclusion (Change 3) ---
stackMgr.SetMigrationDeps(sett, func() bool { return backupMgr != nil && backupMgr.IsRunning() })
if backupMgr != nil {
backupMgr.SetMigrationRunningCheck(stackMgr.IsMigrating)
}
// RecoverMigration is started AFTER the web server's done-hook is wired (below), so a resumed
// decommission-migration still finalizes the source decommission on completion.
// --- Initialize alert manager ---
alertMgr := web.NewAlertManager(logger)
// --- Initialize notifier ---
notifier := notify.New(cfg.Hub.URL, cfg.Hub.APIKey, cfg.Customer.ID, sett, logger, cfg.Logging.Level == "debug")
// R-97a: wire the quiesce loop's hub-event seam. It MUST happen here and not in
// startQuiesceLoop, because the notifier is constructed after it — and it must happen at all,
// or the seam is inert (the "built but never wired" trap, four recorded instances). Reachability
// is covered by TestQuiesceTierNotifierIsWired in this package.
if quiesceLoop != nil {
quiesceLoop.SetTierNotifier(quiesceTierNotifier{n: notifier})
}
// --- Initialize the app-email SMTP shim (mailrelay) ---
// In-process shim: apps → shim → hub → Resend (the Resend key stays hub-side). It runs only
// when the controller has a hub (URL+key) AND the operational kill-switch is on; the runtime
// global app-email toggle then decides whether it actually listens (Lifecycle.Apply). Listeners
// bind to the app Docker network only — never published to the host/internet.
var mailShim *mailrelay.Lifecycle
if cfg.Hub.URL != "" && cfg.Hub.APIKey != "" && cfg.MailRelay.HardEnabled() {
mailShim = mailrelay.NewLifecycle(mailrelay.Options{
PlainAddr: cfg.MailRelay.PlainListen,
TLSAddr: cfg.MailRelay.TLSListen,
PlainNoTLSAddr: cfg.MailRelay.PlainNoTLSListen,
ServiceName: cfg.MailRelay.ShimHost,
Policy: mailrelay.NewPolicy(cfg.MailRelay.FromDomains),
Forwarder: mailrelay.NewHubForwarder(cfg.Hub.URL, cfg.Hub.APIKey),
Logger: logger,
})
if err := mailShim.Apply(sett.AppEmailEnabled()); err != nil {
logger.Printf("[ERROR] [mailrelay] could not start app-email shim: %v", err)
}
} else {
logger.Printf("[INFO] [mailrelay] app-email shim unavailable (no hub configured or disabled in config)")
}
// --- Initialize self-updater ---
var updater *selfupdate.Updater
if cfg.SelfUpdate.Enabled {
// Phase 1: the host agent performs the container swap. Build a (nil-able) agent client from the
// provisioned local-API config; when absent (un-provisioned guest) self-update is unavailable.
var swapAgent selfupdate.AgentSwapper
if cfg.LocalAPI.Endpoint != "" && cfg.LocalAPI.Token != "" {
if ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint); err != nil {
logger.Printf("[WARN] Self-update: agent client init failed (%v) — updates unavailable", err)
} else {
swapAgent = ac
}
}
updater = selfupdate.NewUpdater(&cfg.SelfUpdate, &cfg.Git, Version, cfg.Paths.DataDir, swapAgent, logger, cfg.Logging.Level == "debug")
updater.SetBackupRunningCheck(func() bool {
return backupMgr != nil && backupMgr.IsRunning()
})
// Check for post-update state (did a previous update succeed or fail?)
if state := updater.VerifyStartup(); state != nil {
notifier.NotifyControllerUpdated(state.PreviousVersion, state.TargetVersion, state.Status == "success")
}
logger.Printf("[INFO] Self-update enabled (check every %s, auto-update: %v, auto-update time: %s)",
cfg.SelfUpdate.CheckInterval, cfg.SelfUpdate.AutoUpdate, cfg.SelfUpdate.AutoUpdateTime)
}
// SLICE 2: the offsite apply-bridge — on startup (async, non-blocking) reconcile the hub-served
// offsite descriptor into a configured key-only offbox target (fail-safe, idempotent, no blind
// TOFU). The config_refresh self-restart re-runs this after a descriptor change (new process →
// startup). Launched HERE (after the self-updater is built) so the R-71a settle-gate can read the
// updater's floor/update-running state and defer the one-time-password consume past a managed
// day-0 floor-update (the F10 race). The gate is wired ONLY when an updater exists — with no update
// mechanism there is no floor-update to race, so the bridge reconciles immediately (Settle nil).
if backupMgr != nil && cfg.Offsite.Enabled && cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
bridge := &offsiteapply.Bridge{
Cfg: cfg,
Consumer: offsiteapply.HTTPConsumer{HubURL: cfg.Hub.URL, CustomerID: cfg.Customer.ID, APIKey: cfg.Hub.APIKey},
Scanner: offsiteapply.KeyscanScanner{},
KeyGen: offsiteapply.ED25519KeyGen{},
Installer: offsiteapply.SSHCopyIDInstaller{},
Prober: offsiteapply.SFTPKeyAuthProber{KeyPath: filepath.Join(cfg.Paths.DataDir, "offbox", "ssh_key")},
Enabler: offsiteapply.EnablerFunc(func(ctx context.Context, host, user string, port int, repoPath, priv, kh string, quotaGB int) error {
tgt := &settings.OffboxTarget{Enabled: true, Host: host, User: user, Port: port, RepoPath: repoPath, Schedule: "daily", QuotaGB: quotaGB}
stage := func(ctx context.Context, pw string) error {
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
if err != nil {
return err
}
return ac.StageEscrowSecret(ctx, pw)
}
return backupMgr.ApplyOffsiteTarget(ctx, tgt, priv, kh, stage)
}),
MarkerPath: filepath.Join(cfg.Paths.DataDir, "offbox", "applied_marker"),
Logger: logger,
}
if updater != nil {
// Thin adapter over the updater's OWN knowledge (StackDataProvider pattern) — the bridge
// never fetches the floor a second way. floorKnown = the floor has been learned from a
// report ACK yet (GetFloor() != "").
u := updater
bridge.Settle = offsiteapply.SettleFunc(func() (string, string, bool, bool) {
floor := u.GetFloor()
return Version, floor, u.IsUpdateRunning(), floor != ""
})
}
go func() {
// ReconcileWhenSettled runs the settle-gate FIRST (its own bounds), then Reconcile under a
// fresh 3-minute context — the gate's wait never eats the reconcile budget.
if err := bridge.ReconcileWhenSettled(context.Background()); err != nil {
logger.Printf("[WARN] [offsite-apply] reconcile: %v (retries on next config refresh/restart)", err)
}
}()
}
// --- Initialize scheduler ---
sched := scheduler.New(logger)
sched.SetDebug(cfg.Logging.Level == "debug")
// Existing periodic tasks (migrated from ad-hoc goroutines)
// F7: 30s left the dashboard list lagging Docker health by up to ~30s after a deploy/state change.
// 10s (matching the health-probes cadence) tightens it; RefreshStatus is a cheap `docker ps`-based
// refresh of the in-memory map, so 10s does not meaningfully load Docker. (The deploy page itself
// already polls per-stack every 3s; this is for the dashboard/stacks list.)
sched.Every("status-refresh", 10*time.Second, func(ctx context.Context) error {
return stackMgr.RefreshStatus()
})
sched.Every("stack-scan", 2*time.Minute, func(ctx context.Context) error {
return stackMgr.ScanStacks()
})
sched.Every("health-probes", 10*time.Second, func(ctx context.Context) error {
return stackMgr.RunHealthProbes()
})
// System health check — refreshes dashboard alerts and notifies on changes.
// Healthchecks.io pinging has been retired (the Hub now owns monitoring).
healthInterval, err := time.ParseDuration(cfg.Monitoring.SystemHealthInterval)
if err != nil {
healthInterval = 5 * time.Minute
}
sched.Every("system-health", healthInterval, func(ctx context.Context) error {
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), sett.GetSMBSettings(), logger)
// Self-heal the base stack: call unconditionally every tick. EnsureBaseStack is single-flight
// + idempotent (skips running stacks ⇒ a cheap 3× docker-inspect no-op when healthy), so there
// is no need to couple to the health-report issue strings. Runs in a goroutine — never blocks
// or fails the health job.
go func() {
if err := stackMgr.EnsureBaseStack(); err != nil {
logger.Printf("[WARN] [infra] self-heal base-stack bring-up: %v", err)
}
}()
// Refresh dashboard alerts from health report
updateAvailable := false
latestVersion := ""
if updater != nil {
status := updater.GetStatus()
if status.LastCheck != nil {
updateAvailable = status.LastCheck.UpdateAvailable
latestVersion = status.LastCheck.LatestVersion
}
}
alertMgr.Refresh(healthReport, cfg, backupMgr, updateAvailable, latestVersion, sett.GetStoragePaths())
// Notify on health status changes
notifier.NotifyHealthChange(healthReport.Status, healthReport.Issues, healthReport.Warnings)
return nil
})
// fix-3 (CAMPAIGN-3): a deployed app that is not running must be LOUD, not silent (the campaign's
// CWA sat dead 4 h; F11 then produced 4 silently-dead NAS apps per reboot). Runs on its own short
// cadence (faster than the 5-min health cycle) so a dead app surfaces quickly. Logs nothing on the
// quiet path (no ring spam — fix-6 friendly); the dashboard banner is state-based (self-clears) and
// the hub event fires once per running→down transition (notifier tracks it). A boot grace skips the
// controller's own startup settle so apps that legitimately take 3060 s to come up don't alert.
sched.Every("deadapp-check", 30*time.Second, func(ctx context.Context) error {
if time.Since(startTime) < deadAppBootGrace {
return nil // still inside the startup settle window
}
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop)
alertMgr.SetDeadAppAlerts(dead)
notifier.NotifyAppStartFailures(states)
return nil
})
// fix-6 (CAMPAIGN-3): periodically spill the debug ring to the SSD state dir so a HARD crash loses
// at most ~60 s of the window (a clean shutdown spills too). ≤30s interval → auto-quiet (no
// per-cycle scheduler line). SpillTo is atomic (tmp+rename) so it can never corrupt the ring file.
sched.Every("ring-spill", 30*time.Second, func(ctx context.Context) error {
return logBuffer.SpillTo(ringSpillPath)
})
// --- Central hub pusher (declared early so backup closure can reference it) ---
var hubPusher *report.Pusher
// escrowConfirmer is hoisted so the web server (built later) can read its Scenario-F
// stale-blob flag (SetEscrowStale below). nil when no hub is configured.
var escrowConfirmer *report.EscrowAutoConfirmer
if cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
hubPusher = report.NewPusher(&cfg.Hub, logger, cfg.Logging.Level == "debug")
// SLICE 3 — hub-verified escrow auto-confirm (long-lived: the mismatch warn dedupes per hash,
// not per 15-min cycle). Flips offbox pending→escrowed ONLY when the hub-recorded hash of the
// escrowed password matches the local repo password's hash; never un-confirms. v0.127.0 adds
// the escrowed-state STALE re-check (Scenario F — warn + card flag, never a state change).
escrowConfirmer = &report.EscrowAutoConfirmer{
Pending: func() bool {
return backupMgr != nil && backupMgr.OffboxConfigured() &&
sett.GetOffboxTarget() != nil && sett.GetOffboxTarget().EscrowState == "pending"
},
Escrowed: func() bool {
return backupMgr != nil && backupMgr.OffboxConfigured() &&
sett.GetOffboxTarget() != nil && sett.GetOffboxTarget().EscrowState == "escrowed"
},
LocalHash: func() (string, bool) {
if backupMgr == nil {
return "", false
}
return backupMgr.OffboxRepoPasswordHash()
},
Flip: func() error {
return sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
o.EscrowState = "escrowed"
o.CeremonyCompletedAt = "" // v0.138.0: clear the awaiting-card stamp on confirm
})
},
Wipe: func(ctx context.Context) error {
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
if err != nil {
return err
}
return ac.WipeStagedEscrowSecret(ctx)
},
Logger: logger,
}
// Wire hub verification: update settings when hub reports customer status
hubPusher.OnPushResponse = func(resp *report.PushResponse) {
if resp.CustomerBlocked {
sett.SetHubVerified(false, time.Now())
logger.Printf("[WARN] Customer blocked on Hub — new deployments may be restricted")
} else {
sett.SetHubVerified(true, time.Now())
}
// Phase 2 managed updates: the ACK carries the operator-enforced minimum version (FLOOR).
// Hand it to the updater and reconcile — if the box is below the floor it auto-updates to
// the floor (reusing the Phase 1 swap). This rides the existing report cycle; no new timer.
// latest_version stays informational (the customer's opt-in button), NOT the auto-target.
if updater != nil {
updater.SetFloor(resp.MinControllerVersion)
updater.MaybeAutoUpdate()
}
// Config-refresh (v0.26.0): the ACK also carries the per-customer config_version. On a
// change vs. the last-applied version, re-pull controller.yaml (re-merging local_api from
// bootstrap.json) and self-restart so the new config loads. Pull-based — the hub never
// connects into the box. Rides this same report cycle; no new timer, no agent. First-ever
// ACK records the baseline without restarting; a failed pull keeps the current config and
// retries next cycle; an unchanged version is a no-op (so no restart storm).
cr := &report.ConfigRefresher{
Applied: sett.GetAppliedConfigVersion,
Record: sett.SetAppliedConfigVersion,
Refresh: func() error { return bootstrap.RefreshConfig(*configPath, logger, pull) },
Restart: func() { api.GracefulSelfRestart(logger) },
Logger: logger,
}
cr.Reconcile(resp.ConfigVersion)
// SLICE 3: run the escrow auto-confirm on the same ACK (after the config refresh decision —
// a refresh-restart re-enters here anyway on the next cycle).
escrowConfirmer.Reconcile(resp.Escrow)
// v0.111.0: operator log-tail requests ride the same ACK (pull pattern — the hub
// never reaches in). The NEXT report cycle collects + ships the tails; the hub
// clears its pending request on receipt. An empty list clears any stale local set.
report.SetPendingLogTails(resp.LogTailRequests)
// v0.116.0: the controller's OWN ring, same pattern (selftail.go).
report.SetPendingControllerLog(resp.ControllerLogRequested)
// v0.122.0 (F-4): cache the hub-delivered claim-code state (idempotent by
// generation). The web gate reads it on the next request — no restart needed.
claimSync := &report.ClaimSync{Settings: sett, Logger: logger}
claimSync.Reconcile(resp.Claim)
}
// Wire hub push status into alert manager for dashboard alerts
alertMgr.SetHubPushStatus(func() web.HubPushStatusData {
s := hubPusher.GetStatus()
return web.HubPushStatusData{
LastAttempt: s.LastAttempt,
LastSuccess: s.LastSuccess,
LastError: s.LastError,
Consecutive: s.Consecutive,
}
})
}
// Backup daily jobs
if cfg.Backup.Enabled && backupMgr != nil {
// v0.168.0: ONE customer setting (the window start W) drives all three nightly legs at fixed
// offsets — db-dump at W, tier-2 at W+60m, off-box at W+105m — so they can never be misordered.
// The window resolves settings > controller.yaml > "02:30"; a UI save fans out via UpdateDaily.
win := backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
dbLeg, tier2Leg, offboxLeg := backupwindow.LegTimes(win)
// App-data backup: daily database dumps. Disk-tier (restic snapshots,
// cross-drive, integrity check, infra backup) has moved to the host agent.
sched.Daily("db-dump", dbLeg, func(ctx context.Context) error {
err := backupMgr.RunDBDumps(ctx)
if err != nil {
notifier.NotifyDBDumpFailed("Adatbázis mentés sikertelen", err.Error())
} else {
notifier.NotifyDBDumpCompleted(notify.DBDumpDetails{})
}
return err
})
// Cache refresh: every 5 minutes. Recompute the effective window each pass so the cached
// "next DB dump" follows a runtime window change (the UI save also refreshes immediately).
sched.Every("backup-cache", 5*time.Minute, func(ctx context.Context) error {
curDB, _, _ := backupwindow.LegTimes(backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule))
backupMgr.RefreshCache(scheduler.NextDailyRun(curDB))
return nil
})
// Tier 2: off-drive copy of each HDD app's recovery unit + userdata (auto-enabled, auto-target).
// Runs after the DB dump so it copies a fresh unit.
backupMgr.SetTier2Notifier(func(stackName, destLabel string, dur time.Duration, err error) {
// DISPLAY BOUNDARY (R-7b): this event's StackName lands in Hungarian operator/customer copy
// ("Másodlagos mentés elkészült: <név>"), so the reserved shares key is mapped here. Caught
// by live validation — the first demo run pushed a literal "_shares" before this line.
display := backup.DisplayStackName(stackName)
if err != nil {
notifier.NotifyCrossDriveFailed(notify.CrossDriveDetails{
StackName: display, Method: "rsync", DestPath: destLabel,
Duration: dur.Round(time.Second).String(), Error: err.Error(),
})
} else {
notifier.NotifyCrossDriveCompleted(notify.CrossDriveDetails{
StackName: display, Method: "rsync", DestPath: destLabel,
Duration: dur.Round(time.Second).String(),
})
}
})
sched.Daily("tier2-backup", tier2Leg, func(ctx context.Context) error {
backupMgr.RunAllTier2()
return nil
})
// Off-box (NAS) restic-SFTP backup (Part B): the off-site leg. A failure (incl. a fail-fast dead-NAS
// error) alerts the operator via the allowlisted backup_failed event. Daily after Tier 2.
backupMgr.SetOffboxNotify(func(dur time.Duration, snapshots int, err error) {
if err != nil {
notifier.NotifyBackupFailed("Off-box (NAS) mentés sikertelen",
"a NAS-ra mentés hibázott ("+dur.Round(time.Second).String()+"): "+err.Error())
}
})
// 3a: the pre-push enlargement gate blocked an app's userdata push (config+DB still saved). Edge-
// triggered by the engine (only NEW blocks notify), so the hub's per-event-type cooldown suffices —
// no controller-side timer (the hub owns cooldown).
backupMgr.SetOffboxEnlargeBlockedNotifier(func(stack string, estBytes int64, usedGB, quotaGB int) {
notifier.NotifyOffboxEnlargeBlocked(fmt.Sprintf(
"A(z) %s teljes távoli mentése (~%s) túllépné a tárhelykeretet (%d/%d GB). A konfiguráció és az adatbázis továbbra is mentésre kerül; nagyobb kerethez vedd fel velünk a kapcsolatot.",
stack, appbackup.HumanizeBytes(estBytes), usedGB, quotaGB))
})
// v0.142.0 offsite-repo continuity: push a hub event on the orphaned/reset transitions (once per
// transition — the Manager guards nightly re-fire). Operator-visible on the customer page.
backupMgr.SetOffboxOrphanEvent(func(eventType, renamedTo string) {
switch eventType {
case "offbox_repo_orphaned":
notifier.PushEvent("offbox_repo_orphaned", "warning",
"A távoli mentési tároló elárvult: a benne lévő mentések egy korábbi, már nem elérhető kulccsal készültek (újratelepítés). Új mentés a tároló visszaállításáig nem készül.", nil)
case "offbox_repo_reset":
notifier.PushEvent("offbox_repo_reset", "info",
"A távoli mentési tároló visszaállítva: a régi előzmény félretéve (nem törölve), és egy üres, új tároló jött létre a mostani kulccsal.", map[string]string{"renamed_to": renamedTo})
}
})
sched.Daily("offbox-backup", offboxLeg, func(ctx context.Context) error {
t := sett.GetOffboxTarget()
if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
return nil // not configured / not scheduled
}
return backupMgr.RunOffboxBackup(ctx)
})
}
// Metrics prune — daily at 04:00
if metricsStore != nil {
sched.Daily("metrics-prune", "04:00", func(ctx context.Context) error {
deleted, err := metricsStore.Prune(30 * 24 * time.Hour)
if err != nil {
return err
}
logger.Printf("[INFO] Pruned %d old metric rows", deleted)
return nil
})
}
// --- Central hub reporting schedule ---
if hubPusher != nil {
if cfg.Hub.Enabled {
pushInterval, err := time.ParseDuration(cfg.Hub.PushInterval)
if err != nil {
pushInterval = 15 * time.Minute
}
sched.Every("hub-report", pushInterval, func(ctx context.Context) error {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
if err := hubPusher.Push(r); err != nil {
return err
}
// Drain pending events (e.g., DR recovery completed) after successful push
if events := sett.DrainPendingEvents(); len(events) > 0 {
for _, ev := range events {
notifier.Notify(ev.EventType, ev.Severity, ev.Message, ev.Details)
}
logger.Printf("[INFO] Drained %d pending events to Hub", len(events))
}
return nil
})
logger.Printf("[INFO] Hub reporting enabled (every %s to %s)", pushInterval, cfg.Hub.URL)
} else {
logger.Printf("[INFO] Hub reporting disabled — will send disabled notification to %s", cfg.Hub.URL)
}
}
// Self-update scheduler jobs
if cfg.SelfUpdate.Enabled && updater != nil {
// Periodic version check (populates UI, never triggers update)
checkInterval, ciErr := time.ParseDuration(cfg.SelfUpdate.CheckInterval)
if ciErr != nil {
checkInterval = 6 * time.Hour
}
sched.Every("selfupdate-check", checkInterval, func(ctx context.Context) error {
result := updater.CheckForUpdate()
if result.UpdateAvailable {
logger.Printf("[INFO] Update available: %s -> %s", result.CurrentVersion, result.LatestVersion)
}
return nil
})
// Auto-update (daily, fires after typical backup completion)
if cfg.SelfUpdate.AutoUpdate {
sched.Daily("selfupdate-auto", cfg.SelfUpdate.AutoUpdateTime, func(ctx context.Context) error {
result := updater.CheckForUpdate()
if !result.UpdateAvailable {
return nil
}
if err := updater.TriggerUpdate("auto"); err != nil {
logger.Printf("[WARN] Auto-update skipped: %v", err)
}
return nil
})
}
}
// Storage watchdog (disk disconnect/reconnect detection) has moved to the host
// agent (slice 8C) — the controller no longer owns disk-level monitoring.
// --- Asset syncer (download from Hub) ---
var assetsSyncer *assets.Syncer
if cfg.Hub.Enabled && cfg.Assets.SyncEnabled && cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
assetsDir := filepath.Join(cfg.Paths.DataDir, "assets")
assetsSyncer = assets.New(cfg.Hub.URL, cfg.Hub.APIKey, assetsDir, "/usr/share/felhom/assets", logger, cfg.Logging.Level == "debug")
go func() {
time.Sleep(10 * time.Second)
if err := assetsSyncer.Sync(ctx); err != nil {
logger.Printf("[WARN] Initial asset sync failed: %v", err)
}
}()
sched.Daily("asset-sync", cfg.Assets.SyncSchedule, func(ctx context.Context) error {
return assetsSyncer.Sync(ctx)
})
logger.Printf("[INFO] Asset sync enabled (daily at %s from Hub)", cfg.Assets.SyncSchedule)
}
// --- Startup self-test ---
selfTestResult := selftest.Run(cfg, sett, logger)
sched.Start(ctx)
defer sched.Stop()
// Generate recovery info file if retrieval password is set
if rp := sett.GetRetrievalPassword(); rp != "" {
go func() {
info := recovery.Info{
CustomerID: cfg.Customer.ID,
RetrievalPassword: rp,
HubURL: cfg.Hub.URL,
SupportEmail: "support@felhom.eu",
SupportURL: "https://felhom.eu/kapcsolat",
}
if err := recovery.GenerateRecoveryFile(info, Version, cfg.Paths.DataDir); err != nil {
logger.Printf("[WARN] Failed to generate recovery-info.txt: %v", err)
}
}()
}
// Fire startup pings + hub report immediately (don't wait for first scheduler tick)
go func() {
time.Sleep(5 * time.Second) // Let all subsystems fully initialize
// Push controller startup event to Hub
notifier.NotifyControllerStarted(Version, map[string]interface{}{
"selftest_pass": selfTestResult.Pass,
"selftest_warn": selfTestResult.Warn,
"selftest_fail": selfTestResult.Fail,
})
// Hub report
if hubPusher != nil {
if cfg.Hub.Enabled {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
var pushErr error
for attempt := 1; attempt <= 3; attempt++ {
pushErr = hubPusher.Push(r)
if pushErr == nil {
logger.Println("[INFO] Startup hub report sent")
break
}
logger.Printf("[WARN] Startup hub report attempt %d/3 failed: %v", attempt, pushErr)
if attempt < 3 {
time.Sleep(15 * time.Second)
}
}
if pushErr != nil {
logger.Printf("[WARN] Startup hub report failed after 3 attempts — next scheduled push in %s", cfg.Hub.PushInterval)
}
} else {
// Send a minimal "disabled" notification so hub knows reporting is intentionally off
r := &report.Report{
Version: 1,
CustomerID: cfg.Customer.ID,
CustomerName: cfg.Customer.Name,
ControllerVersion: Version,
Timestamp: time.Now().UTC(),
ReportingDisabled: true,
Health: report.HealthReport{Status: "disabled", Issues: []string{}, Warnings: []string{}},
Stacks: report.StacksReport{Deployed: []string{}, Available: []string{}},
Containers: report.ContainerReport{List: []report.ContainerDetailReport{}},
}
hubPusher.PushOnce(r)
}
}
// Initial self-update check (so settings page shows version info quickly)
if updater != nil {
time.Sleep(25 * time.Second) // Additional delay after hub report
result := updater.CheckForUpdate()
if result.UpdateAvailable {
logger.Printf("[INFO] Startup: update available %s -> %s", result.CurrentVersion, result.LatestVersion)
} else if result.Error != "" {
logger.Printf("[DEBUG] Startup version check: %s", result.Error)
}
}
}()
// Initial backup cache population (don't block startup)
if cfg.Backup.Enabled && backupMgr != nil {
go func() {
curDB, _, _ := backupwindow.LegTimes(backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule))
backupMgr.RefreshCache(scheduler.NextDailyRun(curDB))
}()
}
// Sync notification preferences to hub on startup (handles hub DB rebuild recovery)
if notifier.IsEnabled() {
go func() {
prefs := sett.GetNotificationPrefs()
if prefs.Email != "" {
if err := notifier.SyncPreferences(prefs.Email, prefs.EnabledEvents, prefs.CooldownHours); err != nil {
logger.Printf("[WARN] Failed to sync notification preferences on startup: %v", err)
}
}
}()
}
// Initial alert refresh (so alerts appear immediately, not after first 5min health check)
go func() {
report := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), sett.GetSMBSettings(), logger)
alertMgr.Refresh(report, cfg, backupMgr, false, "")
}()
// --- Out-of-cycle report trigger (v0.139.0, Direction 1) ---
// ONE canonical fire closure (full BuildReport + Claimed + Push) behind a debounced,
// coalescing trigger — user actions with hub-side effects (geo, escrow claim, settings
// save, offsite toggle, app deploy/remove, customer claim) round-trip in seconds instead
// of the next ~15-min cycle. The scheduled hub-report job above stays the reconciliation
// backbone; the trigger is best-effort on top (its failures degrade to the cycle).
// nil when hub reporting is off → the api/web seams stay unset (strict no-op).
var reportTrigger *report.Trigger
if hubPusher != nil && cfg.Hub.Enabled {
fireReport := func() error {
rep := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
rep.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
return hubPusher.Push(rep)
}
reportTrigger = report.NewTrigger(fireReport, logger)
go reportTrigger.Run(ctx)
// Direction-2 immediate-sync (v0.140.0): hold a hanging GET against the hub's wait channel
// and fire the trigger the instant operator intent moves — so an operator save round-trips in
// seconds instead of on the next ~15-min cycle. Reuses the SAME hub URL + key as the pusher;
// no new config keys. Gated on the trigger existing (hub reporting enabled). The ACK from the
// fired report delivers everything through the unchanged machinery.
waiter := report.NewWaiter(cfg.Hub.URL, cfg.Hub.APIKey, reportTrigger.Fire, logger)
go waiter.Run(ctx)
}
// --- Initialize API router ---
apiRouter := api.NewRouter(cfg, *configPath, sett, stackMgr, syncer, cpuCollector, backupMgr, metricsStore, updater, notifier, logger)
if reportTrigger != nil {
// Out-of-cycle, non-blocking hub report push (geo changes + app deploy/remove) so
// the hub reflects the new state immediately instead of after the next ~15-min cycle.
apiRouter.SetReportPushTrigger(reportTrigger.Fire)
}
if assetsSyncer != nil {
apiRouter.SetAssetsSyncer(assetsSyncer)
}
// --- Initialize Cloudflare geo-restriction ---
var geoSync *cf.GeoSyncManager
if cfg.Infrastructure.CFAPIToken != "" {
cfClient := cf.New(cfg.Infrastructure.CFAPIToken, logger, cfg.Logging.Level == "debug")
geoStacks := &geoStackAdapter{mgr: stackMgr, domain: cfg.Customer.Domain}
geoSync = cf.NewGeoSyncManager(cfClient, sett, cfg.Customer.Domain, geoStacks, logger)
geoSync.SetDebug(cfg.Logging.Level == "debug")
apiRouter.SetGeoSync(geoSync)
// Re-sync geo rules when apps are deployed/removed
apiRouter.OnGeoRelevantChange = func() {
geo := sett.GetGeoRestriction()
if geo != nil && geo.Enabled {
if err := geoSync.Sync(context.Background()); err != nil {
logger.Printf("[WARN] Geo sync after app change failed: %v", err)
}
}
}
// Periodic verification every 6 hours
sched.Every("geo-verify", 6*time.Hour, func(ctx context.Context) error {
geo := sett.GetGeoRestriction()
if geo == nil || !geo.Enabled {
return nil
}
return geoSync.Sync(ctx)
})
// Initial sync (delayed, non-blocking)
go func() {
time.Sleep(15 * time.Second)
if geo := sett.GetGeoRestriction(); geo != nil && geo.Enabled {
if err := geoSync.Sync(context.Background()); err != nil {
logger.Printf("[WARN] Initial geo sync failed: %v", err)
}
}
}()
logger.Printf("[INFO] Geo-restriction support enabled (CF API token configured)")
}
// --- Initialize integration manager ---
integrationStacks := &integrationStackAdapter{mgr: stackMgr}
integrationMgr := integrations.NewManager(sett, integrationStacks, cfg.Customer.Domain, cfg.Paths.StacksDir, encKey, logger)
integrationMgr.SetDebug(cfg.Logging.Level == "debug")
apiRouter.SetIntegrationManager(integrationMgr)
// --- Initialize app exporter ---
exportProv := &exportAdapter{mgr: stackMgr, encKey: encKey}
appExporter := appexport.NewExporter(exportProv, logger, Version)
appExporter.SetDebug(cfg.Logging.Level == "debug")
apiRouter.SetDebug(cfg.Logging.Level == "debug")
// --- Initialize web server ---
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, sched, sett, alertMgr, notifier, updater, logger, Version)
// Migration done-hook: a decommission-initiated migration finalizes the source decommission on
// success (soft-mark + agent). Wire it before RecoverMigration so a resumed one still finalizes.
stackMgr.SetMigrationDoneHook(webServer.OnMigrationDone)
stackMgr.RecoverMigration(ctx)
webServer.SetEncryptionKey(encKey)
webServer.SetAppExporter(appExporter)
// Disk-health degradation check (v0.169.0): every 6h, compare each physical disk's SMART verdict
// against the in-memory baseline and emit disk_health_degraded on a degradation only (first run
// baselines silently; recovery/UNKNOWN never notify). Only on a provisioned guest (an agent to
// read /disks from); the check no-ops gracefully if the agent is unreachable.
if cfg.LocalAPI.Endpoint != "" {
sched.Every("disk-health-check", 6*time.Hour, webServer.RunDiskHealthCheck)
}
// Browser .fab upload (v0.128.0): upload state is in-memory, so a restart strands the .part —
// GC stray part files in every registered drive's exports dir at startup.
webServer.CleanupStaleUploadParts()
// Escrow wizard (v0.127.0): the Scenario-F stale-blob flag feeds the Távoli mentés card.
if escrowConfirmer != nil {
webServer.SetEscrowStale(escrowConfirmer.StaleBlob)
}
webServer.SetIntegrationManager(integrationMgr)
if reportTrigger != nil {
// Out-of-cycle report push after hub-relevant user actions (escrow claim, settings
// save, offsite config/toggle, customer claim) — same debounced trigger as the api
// router's; nil (hub reporting off) leaves the seam a strict no-op.
webServer.SetReportTrigger(reportTrigger.Fire)
}
if quiesceLoop != nil {
webServer.SetBackupTrigger(quiesceLoop) // "Mentés most" → app-consistent backup via the quiesce loop
}
if mailShim != nil {
webServer.SetMailShim(mailShim) // settings toggle starts/stops the app-email shim at runtime
}
if assetsSyncer != nil {
webServer.SetAssetsSyncer(assetsSyncer)
}
if hubPusher != nil {
webServer.SetHubPushStatus(func() web.HubPushStatusData {
s := hubPusher.GetStatus()
return web.HubPushStatusData{
LastAttempt: s.LastAttempt,
LastSuccess: s.LastSuccess,
LastError: s.LastError,
Consecutive: s.Consecutive,
}
})
}
if logBuffer != nil {
webServer.SetLogBuffer(logBuffer)
}
webServer.SetStartTime(startTime)
// Controller→agent channel health (self-health slice). A ~60s probe of the local-API channel via
// the PRODUCTION memoized client (webServer.ProbeAgentChannel), classified + debounced, alerting the
// operator + dashboard on a transition. Only on a provisioned guest (endpoint set) — mirrors
// probeLocalAPI's guard. Registered after webServer (it owns the memoized client); sched.Every
// launches the job immediately since the scheduler is already started.
if cfg.LocalAPI.Endpoint != "" {
chSink := channelSink{notifier: notifier, alertMgr: alertMgr}
chChecker := channelhealth.New(webServer.ProbeAgentChannel, chSink, logger)
sched.Every("agent-channel-health", 60*time.Second, chChecker.Check)
}
// local_api endpoint drift (R-77, from the 2026-07-25 outage): controller.yaml and bootstrap.json
// can disagree indefinitely and silently — the island migration rewrote the latter and the
// controller kept dialling the former for 17.5 h, alerting only "agent unreachable". This NAMES
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins).
//
// Startup-only is sufficient and correct: both files are read at boot and neither changes under a
// running controller, so a periodic re-check would add noise without adding signal.
if d := bootstrap.DetectEndpointDrift(*configPath, cfg, logger); d != nil {
alertMgr.SetEndpointDriftAlert(true, d.HungarianMessage())
if notifier != nil {
notifier.NotifyEndpointDrift(d.EnglishMessage(), d.FingerprintAgrees)
}
}
// Wire debug callbacks (only in debug mode)
if cfg.Logging.Level == "debug" {
dc := &web.DebugCallbacks{}
if hubPusher != nil {
dc.TriggerHubReportPush = func() error {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
return hubPusher.Push(r)
}
}
dc.HubConnectivityTest = func() (int, int64, error) {
start := time.Now()
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(cfg.Hub.URL + "/healthz")
latency := time.Since(start).Milliseconds()
if err != nil {
return 0, latency, err
}
resp.Body.Close()
return resp.StatusCode, latency, nil
}
if cfg.Git.RepoURL != "" {
dc.GiteaConnectivityTest = func() (int, int64, error) {
start := time.Now()
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Head(cfg.Git.RepoURL)
latency := time.Since(start).Milliseconds()
if err != nil {
return 0, latency, err
}
resp.Body.Close()
return resp.StatusCode, latency, nil
}
}
dc.GetTelemetryPreview = func() ([]report.AppTelemetry, error) {
return report.BuildAppTelemetryForDebug(stackMgr, metricsStore, logger), nil
}
webServer.SetDebugCallbacks(dc)
}
// Drive migration (full-drive move) has moved to the host agent (slice 8C);
// the controller no longer runs a DriveMigrator.
// --- Build HTTP mux ---
mux := http.NewServeMux()
// API routes (no auth for health endpoint, auth for everything else)
mux.HandleFunc("/api/health", apiRouter.HealthHandler)
// Disk management API — thin proxy to the host agent (slice 8C). The agent owns
// disk execution; the controller forwards list/assign/eject/format.
mux.Handle("/api/disks", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
mux.Handle("/api/disks/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
// Guided storage provisioning (init/attach/eject orchestration over the agent disk API + registry).
mux.Handle("/api/storage/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeStorageAPI))))
// LAN network-sharing folder picker (R-7 slice 1). Must live here, not in the web ServeHTTP
// switch: the /api/ subtree is routed on this mux, so a case there is shadowed and 401s.
mux.Handle("/api/sharing/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeSharingAPI))))
// Guest RAM resize (v0.143.0, R-24): read current allocation/bounds + apply a bounded resize.
mux.Handle("/api/system/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeSystemAPI))))
// Standalone full-server (guest) restart — the "Kiszolgáló újraindítása" maintenance affordance,
// a sibling to the controller-only /api/selfrestart. Reuses the agent GuestReboot primitive.
mux.Handle("/api/server/reboot", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.HandleServerReboot))))
// Whole-guest (appliance) backup visibility + manual trigger. Distinct prefix from apiRouter's
// app-data /api/backup/{run,status} (DB dumps) to avoid shadowing the /api/ catch-all subtree.
mux.Handle("/api/guest-backup/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeBackupAPI))))
// Host metrics API — thin proxy to the host agent (slice 9). Read-only host-wide health +
// per-storage capacity for the monitoring view; the de-privileged controller can't read the
// host itself. GET only, so no CSRF wrapper needed.
mux.Handle("/api/host-metrics", webServer.RequireAuth(http.HandlerFunc(webServer.ServeHostMetricsAPI)))
// App export/import API routes handled by web server
mux.Handle("/api/export/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeExportAPI))))
// Debug API routes handled by web server (debug-mode gating inside handler)
mux.Handle("/api/debug/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDebugAPI))))
// Escrow ceremony wizard API (v0.127.0) — session auth + CSRF on POSTs; the claim response is
// the ONLY surface the recovery code R ever crosses (no-store, never logged).
mux.Handle("/api/escrow/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeEscrowAPI))))
// Self-update API — accepts session auth OR hub API key (for external triggering)
// CsrfProtect exempts Bearer-token requests automatically.
mux.Handle("/api/selfupdate/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
// Config API — accepts session auth OR hub API key (for Hub config push)
mux.Handle("/api/config/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
// Geo API — accepts session auth OR hub API key (for Hub geo-disable)
mux.Handle("/api/geo/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
mux.Handle("/api/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
// Web UI routes (auth required)
mux.Handle("/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeHTTP))))
// --- Start HTTP server ---
server := &http.Server{
Addr: cfg.Web.Listen,
Handler: webServer.CatchAllMiddleware(mux),
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
}
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigCh
logger.Printf("[INFO] Received signal %v, shutting down...", sig)
// fix-6: spill the debug ring on a clean shutdown so a `systemctl restart` / container recreate
// preserves the pre-restart window (the periodic spill already covers a hard crash to ≤60 s).
if err := logBuffer.SpillTo(ringSpillPath); err != nil {
logger.Printf("[WARN] debug-ring spill on shutdown failed: %v", err)
}
cancel()
if mailShim != nil {
mailShim.Close()
}
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second)
defer shutdownCancel()
if err := server.Shutdown(shutdownCtx); err != nil {
logger.Printf("[ERROR] HTTP server shutdown error: %v", err)
}
}()
logger.Printf("[INFO] Web UI listening on %s", cfg.Web.Listen)
if err := server.ListenAndServe(); err != http.ErrServerClosed {
logger.Fatalf("[FATAL] HTTP server error: %v", err)
}
logger.Println("[INFO] felhom-controller stopped")
}
// selfUpdateAuthMiddleware allows access via session auth (normal UI) OR hub API key bearer token (external).
// deadAppBootGrace is the startup settle window before fix-3 evaluates deployed-app run states — apps
// legitimately take 3060 s to come up, so a shorter grace would false-alarm during the controller's
// own boot. After the grace, an app that still isn't running alerts (the F11 dead-at-boot case).
const deadAppBootGrace = 90 * time.Second
// bootReconcileSettle lets the initial scan, the first status refresh and the quiesce recovery
// settle before the R-52 sweep decides what "down" means. 5 s + at most one 30 s retry gap keeps
// the whole sweep inside deadAppBootGrace (90 s), which is what makes a successful recovery silent.
var bootReconcileSettle = 5 * time.Second
// bootReconcileFn is the R-52 sweep, a package var purely so the wiring below is testable from
// package main (the v0.154.0 / v0.91.0 lesson: a seam proven only through injection proves the
// component and not the caller).
var bootReconcileFn = func(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) bootrecon.Result {
return bootrecon.New(mgr, logger).Run(ctx)
}
// runBootReconcile waits out the settle window, then performs exactly one bounded recovery sweep.
// Called from main() in a goroutine; returns after the single sweep — there is no loop by design.
func runBootReconcile(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) {
select {
case <-ctx.Done():
return
case <-time.After(bootReconcileSettle):
}
bootReconcileFn(ctx, mgr, logger)
}
// scanDeployedAppRunStates returns the fix-3 view of the deployed apps: the DEAD ones (for the
// state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
// transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks()
// — the derivation itself lives in classifyRunStates so it is testable without a live Manager.
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop) ([]web.DeadApp, []notify.AppRunState) {
// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
return classifyRunStates(mgr.GetStacks(), q.SuppressedStacks())
}
// classifyRunStates is the pure fix-3 derivation over a plain stack slice. It splits the deployed
// apps into the DEAD list (dashboard banner) and the per-app run states (notifier transition tracker).
//
// v0.164.0: a deliberate user stop is NOT a fault and must not alarm anywhere (banner OR email). The
// down predicate therefore EXCLUDES StateStopped, resting on two invariants:
// - I1: the UI stop path Manager.StopStack runs `docker compose down` → containers are removed, and
// a deployed stack with zero containers aggregates to StateStopped (manager.go refreshStatusLocked).
// So StateStopped means "deployed, deliberately stopped by the user".
// - I2: the P2 restart-policy census (2026-07-21, 53 templates / 78 services) found every catalog
// service on `unless-stopped`, so a crashing app never comes to rest at `stopped` — faults surface
// as StateExited / StateDegraded (and restarting/unhealthy). StateStopped is therefore never a fault.
//
// If either invariant changes, revisit this suppression. (An out-of-band `docker compose stop` leaves
// the containers present → StateExited → still alerts, which is correct: out-of-band tampering IS
// reportable.) IsDownState is intentionally left unchanged — other callers rely on stopped counting as
// down; the suppression is a filter at this single derivation point only.
func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool) ([]web.DeadApp, []notify.AppRunState) {
var dead []web.DeadApp
var states []notify.AppRunState
for _, st := range sts {
if !st.Deployed || st.Deploying {
continue
}
// R-97b: a stack a quiesce cycle stopped (or restarted within the grace window) is NOT down —
// we stopped it. This is a CYCLE-keyed suppression, not a state test, because an app caught
// mid-restart is `starting`/`unhealthy`, not StateStopped, so v0.164.0's state filter above
// cannot see it. The window EXPIRES (quiesceAlarmGrace): an app that genuinely fails to come
// back still alarms on the first scan after it closes.
down := stacks.IsDownState(st.State) && st.State != stacks.StateStopped && !quiesced[st.Name]
states = append(states, notify.AppRunState{Name: st.Name, DisplayName: st.Meta.DisplayName, Down: down})
if down {
dead = append(dead, web.DeadApp{Name: st.Name, DisplayName: st.Meta.DisplayName, State: string(st.State)})
}
}
return dead, states
}
func selfUpdateAuthMiddleware(cfg *config.Config, webServer *web.Server, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Check bearer token first (for external API calls: hub, build scripts)
if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "Bearer ") {
token := strings.TrimPrefix(auth, "Bearer ")
if token != "" && cfg.Hub.APIKey != "" && subtle.ConstantTimeCompare([]byte(token), []byte(cfg.Hub.APIKey)) == 1 {
next.ServeHTTP(w, r)
return
}
}
// Fall back to session auth
webServer.RequireAuth(next).ServeHTTP(w, r)
})
}
// setupLogger builds the v0.116.0 capture layer: the LogBuffer ring ALWAYS exists
// and captures every line (DEBUG included — remote diagnostics need the detail to
// exist without a config flip), while stdout (docker logs) keeps respecting
// logging.level via the level filter. Lshortfile stays debug-only (unchanged).
func setupLogger(cfg *config.Config) (*log.Logger, *web.LogBuffer) {
// fix-6 (CAMPAIGN-3): 1000 wrapped in ~6.5 min under the campaign's ~2.6 entries/s load — the exact
// post-incident window an operator needs was the first thing lost. 5000 gives ≈32 min at that raw
// rate, and ≈50+ min now that the periodic-scheduler TRACE noise no longer enters the ring (2b).
logBuffer := web.NewLogBuffer(5000)
flags := log.LstdFlags
if cfg.Logging.Level == "debug" {
flags |= log.Lshortfile
}
stdout := web.NewLevelFilterWriter(os.Stdout, cfg.Logging.Level)
return log.New(io.MultiWriter(stdout, logBuffer), "", flags), logBuffer
}
// stackAdapter implements backup.StackDataProvider using stacks.Manager.
type stackAdapter struct {
mgr *stacks.Manager
getStoragePaths func() []settings.StoragePath
encKey []byte // for decrypting live app.yaml secrets during restore-from-unit
}
func (a *stackAdapter) GetStackComposePath(name string) (string, bool) {
s, ok := a.mgr.GetStack(name)
if !ok {
return "", false
}
return s.ComposePath, true
}
// GetStackClassifiedBinds delegates to the stacks manager's backup-classification resolver (Task 2).
// INERT — no backup tier calls this yet; wired so Task 3 consumes a tested seam.
func (a *stackAdapter) GetStackClassifiedBinds(name string) ([]backup.ClassifiedBind, bool) {
return a.mgr.ClassifiedBinds(name)
}
func (a *stackAdapter) ListDeployedStacks() []backup.StackSummary {
var result []backup.StackSummary
for _, s := range a.mgr.GetStacks() {
if !s.Deployed {
continue
}
result = append(result, backup.StackSummary{
Name: s.Name,
DisplayName: s.Meta.DisplayName,
ComposePath: s.ComposePath,
NeedsHDD: s.Meta.Resources.NeedsHDD,
HasVolumes: len(backup.ParseComposeNamedVolumes(s.ComposePath)) > 0,
})
}
return result
}
func (a *stackAdapter) StopStack(name string) error {
return a.mgr.StopStack(name)
}
func (a *stackAdapter) StartStack(name string) error {
return a.mgr.StartStack(name)
}
func (a *stackAdapter) GetStackHDDMounts(name string) []string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
// Priority 1: Read the app's own HDD_PATH from its app.yaml
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
return stacks.ParseComposeHDDMounts(s.ComposePath, appCfg.Env["HDD_PATH"])
}
// Priority 2: Try all registered storage paths (fallback)
var allMounts []string
seen := make(map[string]bool)
for _, sp := range a.getStoragePaths() {
mounts := stacks.ParseComposeHDDMounts(s.ComposePath, sp.Path)
for _, m := range mounts {
if !seen[m] {
seen[m] = true
allMounts = append(allMounts, m)
}
}
}
return allMounts
}
func (a *stackAdapter) GetDockerVolumes(name string) []string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
return backup.ResolveDockerVolumeNames(s.ComposePath)
}
// GetImportRoot delegates to the stack manager's canonical drop-zone root (R-75). Both the backup
// and the export provider need it: ${IMPORT_PATH} binds live on the SYSTEM drive and can NOT be
// resolved from an app's HDD_PATH.
func (a *stackAdapter) GetImportRoot() string { return a.mgr.GetImportRoot() }
func (a *stackAdapter) GetStackHDDPath(name string) string {
s, ok := a.mgr.GetStack(name)
if !ok {
return ""
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
return filepath.Clean(appCfg.Env["HDD_PATH"])
}
return ""
}
// GetStackRecoveryInfo gathers the SECRET-FREE inputs for an app's recovery unit (Phase 2): the
// stack dir, pinned image tags, the non-secret env, and the NAMES of secret/data-key env vars.
// It deliberately does NOT decrypt or return any secret value — secret/password fields are stored
// encrypted in app.yaml, so excluding them (plus a defensive crypto.IsEncrypted guard) yields a
// plaintext, secret-free env. The actual secret values are recovered at restore time from the
// guest's own app.yaml (live, or via the PBS whole-guest snapshot), never from the unit.
func (a *stackAdapter) GetStackRecoveryInfo(name string) (backup.RecoveryInfo, bool) {
s, ok := a.mgr.GetStack(name)
if !ok {
return backup.RecoveryInfo{}, false
}
stackDir := filepath.Dir(s.ComposePath)
meta := stacks.LoadMetadata(stackDir)
// Secret set = all secret/password fields any data_key fields (in deterministic metadata order).
secretSet := make(map[string]bool)
var secretNames []string
add := func(v string) {
if !secretSet[v] {
secretSet[v] = true
secretNames = append(secretNames, v)
}
}
for _, v := range stacks.SensitiveEnvVars(&meta) {
add(v)
}
dataKeys := meta.DataKeyEnvVars()
for _, v := range dataKeys {
add(v)
}
// Non-secret env: raw app.yaml values that are neither named-secret nor (defensively) encrypted.
nonSecret := make(map[string]string)
if appCfg := stacks.LoadAppConfig(stackDir); appCfg != nil {
for k, v := range appCfg.Env {
if secretSet[k] || crypto.IsEncrypted(v) {
continue
}
nonSecret[k] = v
}
}
return backup.RecoveryInfo{
StackDir: stackDir,
DisplayName: s.Meta.DisplayName,
ImagePins: backup.ParseComposeImages(s.ComposePath),
NonSecretEnv: nonSecret,
SecretEnvVars: secretNames,
DataKeyEnvVars: dataKeys,
}, true
}
// RecoverStackSecrets returns the live decrypted values for the named secret env vars present in the
// stack's app.yaml (the guest's own — live rootfs or PBS-restored). Absent/empty names are omitted;
// the caller's fail-closed gate decides. Secrets come from the guest, never from the recovery unit.
func (a *stackAdapter) RecoverStackSecrets(name string, names []string) map[string]string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
cfg := stacks.LoadAppConfigDecrypted(filepath.Dir(s.ComposePath), a.encKey)
if cfg == nil {
return nil
}
out := make(map[string]string)
for _, n := range names {
if v, ok := cfg.Env[n]; ok && v != "" {
out[n] = v
}
}
return out
}
// RecreateStackDefinitionFromUnit restores the app definition from the unit's compose dir into the
// stack dir and persists app.yaml from the reconstructed full env. Secrets in fullEnv were recovered
// from the guest, never regenerated. It starts NOTHING — the restore flow brings the database service
// up alone for the dump replay and only then starts the whole stack (R-47).
func (a *stackAdapter) RecreateStackDefinitionFromUnit(name, composeSrcDir string, fullEnv map[string]string) error {
s, ok := a.mgr.GetStack(name)
if !ok {
return fmt.Errorf("stack %q not found", name)
}
stackDir := filepath.Dir(s.ComposePath)
// Recover the app definition from the unit (compose + .felhom.yml) into the stack dir.
for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} {
data, err := os.ReadFile(filepath.Join(composeSrcDir, fname))
if err != nil {
continue // capture whichever existed
}
if err := os.WriteFile(filepath.Join(stackDir, fname), data, 0644); err != nil {
return fmt.Errorf("restoring %s from unit: %w", fname, err)
}
}
return a.mgr.PersistUnitRedeployConfig(name, fullEnv)
}
// StartStackServices brings up only the named compose services (the DB-only replay window, R-47).
func (a *stackAdapter) StartStackServices(name string, services []string) error {
return a.mgr.StartStackServices(name, services)
}
// RefreshAndIsRunning forces a docker ps scan before checking state.
// Called during post-restore health check (~every 5s for up to 90s).
// Full refresh is acceptable here since restores are rare operations.
func (a *stackAdapter) RefreshAndIsRunning(name string) bool {
a.mgr.RefreshStatus()
s, ok := a.mgr.GetStack(name)
return ok && s.State == stacks.StateRunning
}
// integrationStackAdapter implements integrations.StackProvider using stacks.Manager.
type integrationStackAdapter struct {
mgr *stacks.Manager
}
func (a *integrationStackAdapter) GetStack(name string) (*stacks.Stack, bool) {
return a.mgr.GetStack(name)
}
func (a *integrationStackAdapter) GetStacks() []stacks.Stack {
return a.mgr.GetStacks()
}
func (a *integrationStackAdapter) RestartStack(name string) error {
return a.mgr.RestartStack(name)
}
// geoStackAdapter implements cloudflare.StackLister for geo-restriction sync.
type geoStackAdapter struct {
mgr *stacks.Manager
domain string
}
func (a *geoStackAdapter) GetDeployedHostnames() map[string]string {
result := make(map[string]string)
for _, stack := range a.mgr.GetStacks() {
if !stack.Deployed {
continue
}
subdomain := stack.Meta.Subdomain
// Check for custom subdomain in app.yaml
if appCfg := a.mgr.LoadAppConfigByName(stack.Name); appCfg != nil {
if sd, ok := appCfg.Env["SUBDOMAIN"]; ok && sd != "" {
subdomain = sd
}
}
if subdomain != "" {
result[stack.Name] = subdomain + "." + a.domain
}
}
return result
}
// exportAdapter implements appexport.ExportStackProvider using stacks.Manager.
type exportAdapter struct {
mgr *stacks.Manager
encKey []byte
}
func (a *exportAdapter) GetStackDir(name string) (string, bool) {
s, ok := a.mgr.GetStack(name)
if !ok {
return "", false
}
return filepath.Dir(s.ComposePath), true
}
// GetStackClassifiedBinds delegates to the stacks manager's backup-classification resolver (Task 2),
// used by the `.fab` class-scoped export plan (Task 4).
func (a *exportAdapter) GetStackClassifiedBinds(name string) ([]appbackup.ClassifiedBind, bool) {
return a.mgr.ClassifiedBinds(name)
}
func (a *exportAdapter) GetStackComposePath(name string) (string, bool) {
s, ok := a.mgr.GetStack(name)
if !ok {
return "", false
}
return s.ComposePath, true
}
func (a *exportAdapter) GetStackHDDMounts(name string) []string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
// C6B-F1 (v0.130.0): union ${HDD_PATH} binds + the ${USERDATA_PATH} root. The old
// ParseComposeHDDMounts-only call was blind to the standard userdata convention, so
// 12/13 needs_hdd catalog apps exported hollow (config-only) bundles. The backup-side
// stackAdapter is intentionally NOT changed here — the scheduled/tier-2 path copies the
// recovery unit + the app's resolved appdata/<name> dir(s) only (NOT the userdata tree —
// F-S1; NOT the namespace wholesale), and derives that dir name from the compose binds
// (F-S2, see backup.tier2AppDataName). Spec:
// felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md.
return stacks.ExportDataMounts(s.ComposePath, appCfg.Env["HDD_PATH"])
}
return nil
}
// GetImportRoot delegates to the stack manager's canonical drop-zone root (R-75). Both the backup
// and the export provider need it: ${IMPORT_PATH} binds live on the SYSTEM drive and can NOT be
// resolved from an app's HDD_PATH.
func (a *exportAdapter) GetImportRoot() string { return a.mgr.GetImportRoot() }
func (a *exportAdapter) GetStackHDDPath(name string) string {
s, ok := a.mgr.GetStack(name)
if !ok {
return ""
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
return filepath.Clean(appCfg.Env["HDD_PATH"])
}
return ""
}
func (a *exportAdapter) IsStackRunning(name string) bool {
s, ok := a.mgr.GetStack(name)
// StateDegraded (R-51) counts as running: the export must stop the still-live members before
// reading their volumes, exactly as it would for a fully running stack.
return ok && (s.State == stacks.StateRunning || s.State == stacks.StateDegraded)
}
func (a *exportAdapter) StopStack(name string) error {
return a.mgr.StopStack(name)
}
func (a *exportAdapter) StartStack(name string) error {
return a.mgr.StartStack(name)
}
func (a *exportAdapter) GetStackDisplayName(name string) string {
s, ok := a.mgr.GetStack(name)
if !ok {
return name
}
return s.Meta.DisplayName
}
func (a *exportAdapter) GetStackNeedsHDD(name string) bool {
s, ok := a.mgr.GetStack(name)
return ok && s.Meta.Resources.NeedsHDD
}
func (a *exportAdapter) GetDockerVolumes(name string) []string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
return backup.ResolveDockerVolumeNames(s.ComposePath)
}
func (a *exportAdapter) IsStackDeployed(name string) bool {
s, ok := a.mgr.GetStack(name)
return ok && s.Deployed
}
func (a *exportAdapter) GetDecryptedEnv(name string) map[string]string {
s, ok := a.mgr.GetStack(name)
if !ok {
return nil
}
stackDir := filepath.Dir(s.ComposePath)
cfg := stacks.LoadAppConfigDecrypted(stackDir, a.encKey)
if cfg == nil {
return nil
}
return cfg.Env
}
func (a *exportAdapter) GetStacksBaseDir() string {
return a.mgr.GetStacksBaseDir()
}
func (a *exportAdapter) SaveEncryptedAppConfig(stackDir string, env map[string]string) error {
meta := stacks.LoadMetadata(stackDir)
sensitiveVars := stacks.SensitiveEnvVars(&meta)
cfg := &stacks.AppConfig{
Deployed: true,
DeployedAt: time.Now().Format(time.RFC3339),
Env: env,
}
return stacks.SaveAppConfig(stackDir, cfg, a.encKey, sensitiveVars)
}
func (a *exportAdapter) RefreshStacks() error {
return a.mgr.RefreshStatus()
}
func (a *exportAdapter) RemoveStackVolumes(name string) error {
s, ok := a.mgr.GetStack(name)
if !ok {
return fmt.Errorf("stack %q not found", name)
}
stackDir := filepath.Dir(s.ComposePath)
// Build env from decrypted app config
cmdEnv := os.Environ()
appCfg := stacks.LoadAppConfigDecrypted(stackDir, a.encKey)
if appCfg != nil {
for k, v := range appCfg.Env {
cmdEnv = append(cmdEnv, k+"="+v)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", "compose", "down", "--volumes")
cmd.Dir = stackDir
cmd.Env = cmdEnv
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("compose down --volumes: %s — %w", strings.TrimSpace(string(out)), err)
}
return nil
}
// fileExists returns true if the path exists (file or directory).
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// quiesceBackend adapts *agentapi.Client to quiesce.Backend (bool/string, decoupled from the
// agentapi response structs).
type quiesceBackend struct{ c *agentapi.Client }
func (b quiesceBackend) Due(ctx context.Context) (bool, *int64, error) {
r, err := b.c.BackupDue(ctx)
return r.Due, r.AgeSecs, err
}
func (b quiesceBackend) StartBackup(ctx context.Context) (string, error) {
r, err := b.c.StartBackup(ctx)
return r.JobID, err
}
func (b quiesceBackend) BackupStatus(ctx context.Context) (string, error) {
r, err := b.c.BackupStatus(ctx)
return r.Phase, err
}
// ---- R-82: the tiered surface (quiesce.TieredBackend) ------------------------------------
//
// quiesceBackend satisfies quiesce.TieredBackend as well, so the loop schedules per tier when the
// agent supports it. Against a PRE-R-82 agent, Tiers returns quiesce.ErrTiersUnsupported and the
// loop degrades to the untargeted methods above — still taking a backup, never skipping one.
func (b quiesceBackend) Tiers(ctx context.Context) ([]quiesce.BackupTier, error) {
r, err := b.c.BackupTiers(ctx)
if errors.Is(err, agentapi.ErrTiersUnsupported) {
// Translate the transport-layer probe into the loop's vocabulary; the loop keys on this.
return nil, quiesce.ErrTiersUnsupported
}
if err != nil {
return nil, err
}
out := make([]quiesce.BackupTier, 0, len(r.Tiers))
for _, t := range r.Tiers {
out = append(out, quiesce.BackupTier{Target: t.Target, Primary: t.Primary})
}
return out, nil
}
func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
r, err := b.c.BackupDueFor(ctx, target)
// AgeState is passed through RAW; quiesce.ageStateFromWire owns the mapping, including the
// legacy-vs-unknown distinction. An empty string here means a pre-v0.105.0 agent.
return r.Due, r.AgeSecs, r.AgeState, err
}
func (b quiesceBackend) StartBackupFor(ctx context.Context, target string) (string, error) {
r, err := b.c.StartBackupFor(ctx, target)
return r.JobID, err
}
func (b quiesceBackend) BackupStatusFor(ctx context.Context, target string) (string, error) {
r, err := b.c.BackupStatusFor(ctx, target)
return r.Phase, err
}
// quiesceTierNotifier adapts *notify.Notifier to quiesce.TierNotifier (R-97a).
//
// The whole-guest tier had NO route to the hub at all — `internal/quiesce` did not import
// `internal/notify`, so on 2026-07-27 three failed backups and twelve app-stack stop/starts produced
// ZERO `backup_failed` events. The event type was already in the hub's allowlist and
// `NotifyBackupFailed` already existed; only this adapter and its wiring were missing.
//
// The tier is carried in the MESSAGE rather than a new event type, because the hub gates on
// `allowedEventTypes` and a per-tier type would need a hub-side change to be deliverable at all.
// See the cooldown note in REPORT.md: the hub's operator cooldown is keyed
// `customerID + ":" + eventType`, so two tiers failing within an hour share one key — both events
// are STORED, but only the first sends an operator email.
type quiesceTierNotifier struct{ n *notify.Notifier }
func (q quiesceTierNotifier) BackupFailed(tier, message, errMsg string) {
q.n.NotifyWholeGuestBackupFailed(tier, message, errMsg)
}
func (q quiesceTierNotifier) BackupRecovered(tier, message string) {
q.n.NotifyWholeGuestBackupRecovered(tier, message)
}
// startQuiesceLoop wires + starts the slice-8B quiesce loop when the local API is configured and
// quiesce is enabled. It Recovers (restarts stacks left stopped by a mid-quiesce crash) before
// starting the loop goroutine. Non-fatal: any misconfig disables the loop with a log line.
func startQuiesceLoop(ctx context.Context, cfg *config.Config, sett *settings.Settings, stackMgr *stacks.Manager, logger *log.Logger) *quiesce.Loop {
if cfg.LocalAPI.Endpoint == "" || cfg.LocalAPI.Token == "" {
return nil // not a provisioned guest — no agent to back up against
}
if !cfg.Quiesce.QuiesceEnabled() {
logger.Printf("[INFO] [quiesce] disabled by config")
return nil
}
client, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
if err != nil {
logger.Printf("[WARN] [quiesce] disabled (agent client init failed): %v", err)
return nil
}
poll := parseDurationOr(cfg.Quiesce.PollInterval, 5*time.Minute)
statusPoll := parseDurationOr(cfg.Quiesce.StatusPoll, 10*time.Second)
maxQuiesce := parseDurationOr(cfg.Quiesce.MaxQuiesce, 30*time.Minute)
loop := quiesce.New(quiesce.Options{
Backend: quiesceBackend{c: client},
Stacks: stackMgr,
MarkerPath: filepath.Join(cfg.Paths.DataDir, "quiesce-state.json"),
Poll: poll,
StatusPoll: statusPoll,
MaxQuiesce: maxQuiesce,
Logger: logger,
// Window gate (v0.168.0): read the effective window fresh each poll so a customer change takes
// effect without restart. Scheduled cycles run only inside [W+2h, W+6h), with the safety valve.
WindowStartFn: func() string {
return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
},
})
loop.Recover() // crash-safety: restart any stacks stranded-down by a mid-quiesce crash
go loop.Run(ctx)
return loop
}
// parseDurationOr parses a duration string, falling back to def on empty/invalid input.
func parseDurationOr(s string, def time.Duration) time.Duration {
if s == "" {
return def
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return def
}
return d
}
// channelSink adapts the notify.Notifier + web.AlertManager to the channelhealth.Sink seam (kept in
// main.go so the channelhealth package imports neither — no cycle). SetDashboard reflects the current
// state each probe (idempotent); NotifyDown/NotifyRecovered fire only on a transition.
type channelSink struct {
notifier *notify.Notifier
alertMgr *web.AlertManager
}
func (s channelSink) SetDashboard(down bool, _ channelhealth.Reason, msg string) {
s.alertMgr.SetAgentChannelAlert(down, msg)
}
func (s channelSink) NotifyDown(reason channelhealth.Reason, eventType, severity, msg string) {
s.notifier.NotifyAgentChannelDown(string(reason), eventType, severity, msg)
}
func (s channelSink) NotifyRecovered() { s.notifier.NotifyAgentChannelRecovered() }
// probeLocalAPI proves the controller↔agent local-API channel at startup and logs this guest's
// mounts (slice 8A). Non-fatal: it only runs when a local-API endpoint is configured, and any
// error is logged for diagnosis without affecting the controller's boot. The leaf SHA-256 from
// the bootstrap is pinned by the client (fails closed on mismatch).
func probeLocalAPI(cfg *config.Config, logger *log.Logger) {
if cfg.LocalAPI.Endpoint == "" || cfg.LocalAPI.Token == "" {
return
}
client, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
if err != nil {
logger.Printf("[WARN] local-api: client init failed (%v) — channel not verified", err)
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
resp, err := client.Storage(ctx)
if err != nil {
logger.Printf("[WARN] local-api: GET /storage failed (%v) — channel not verified", err)
return
}
logger.Printf("[INFO] local-api: channel up (agent %s) — guest %d, %d mount(s) visible",
cfg.LocalAPI.Endpoint, resp.VMID, len(resp.Mounts))
for _, m := range resp.Mounts {
logger.Printf("[INFO] local-api: mount %s → %s (storage=%s, class=%s, backup=%v)",
m.Key, m.MountPoint, m.Storage, m.Class, m.Backup)
}
}
// runSetupMode starts the setup wizard on dual listeners and blocks until signal.
func runSetupMode(cfg *config.Config, logger *log.Logger) {
ips := setup.DetectLocalIPs()
setup.LogSetupMode(cfg.Customer.Domain, ips, cfg.Web.SetupListen, logger)
setupSrv := setup.NewServer(cfg, cfg.Paths.DataDir, logger, Version)
handler := setupSrv.Handler()
// Health endpoint wrapper (returns setup_mode: true)
healthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"ok": true, "message": "felhom-controller is healthy",
"setup_mode": true, "version": Version,
})
})
// Mux for both listeners
mux := http.NewServeMux()
mux.HandleFunc("/api/health", healthHandler)
mux.Handle("/", handler)
// Start main listener (:8080, behind Traefik for domain access)
mainServer := &http.Server{
Addr: cfg.Web.Listen,
Handler: mux,
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
}
go func() {
logger.Printf("[INFO] Setup wizard (main) listening on %s", cfg.Web.Listen)
if err := mainServer.ListenAndServe(); err != http.ErrServerClosed {
logger.Printf("[ERROR] Main HTTP server error: %v", err)
}
}()
// Start setup-only listener (:8081, direct HTTP for LAN access)
setupServer := &http.Server{
Addr: cfg.Web.SetupListen,
Handler: mux,
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
}
go func() {
logger.Printf("[INFO] Setup wizard (LAN) listening on %s", cfg.Web.SetupListen)
if err := setupServer.ListenAndServe(); err != http.ErrServerClosed {
logger.Printf("[ERROR] Setup HTTP server error: %v", err)
}
}()
// Wait for signal
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
sig := <-sigCh
logger.Printf("[INFO] Received signal %v, shutting down setup wizard...", sig)
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()
mainServer.Shutdown(shutdownCtx)
setupServer.Shutdown(shutdownCtx)
logger.Println("[INFO] Setup wizard stopped")
}
// discoverHDDPaths scans deployed apps' app.yaml for HDD_PATH env values.
func discoverHDDPaths(stacksDir string, logger *log.Logger) []string {
entries, err := os.ReadDir(stacksDir)
if err != nil {
logger.Printf("[WARN] Cannot read stacks dir for HDD path discovery: %v", err)
return nil
}
seen := make(map[string]bool)
var paths []string
for _, e := range entries {
if !e.IsDir() {
continue
}
appCfg := stacks.LoadAppConfig(filepath.Join(stacksDir, e.Name()))
if appCfg == nil || !appCfg.Deployed {
continue
}
if hddPath, ok := appCfg.Env["HDD_PATH"]; ok && hddPath != "" {
cleaned := filepath.Clean(hddPath)
if !seen[cleaned] {
seen[cleaned] = true
paths = append(paths, cleaned)
}
}
}
return paths
}