9dc26459ea
gates / gates (push) Successful in 10s
R-218's declaration half shipped in v0.201.0 and works. Its consume half never existed. Reconcile ran exactly twice per process — at start-up and when the recovery screen drives it — and BOTH fire before the hub has anything staged, because the hub stages in RESPONSE to the declaration those runs precede. Measured on the R-201 re-walk: unlock reconcile 11:43:07, hub staged 11:44:57 saying 'next cycle', a full report cycle ran 11:55:46, still unconsumed at 12:06. A guest command line applied it in 18 seconds — everything correct except the trigger. Bridge.RetryIfDeclared re-runs the SAME reconcile on a 5-minute tick, driven from the box's own published declaration (OffboxReportStatus().State) — the very statement the hub acts on, so the two cannot disagree. Poll, not an ACK flag, decided on the promise: the no-target message says 'amint megvannak' (no deadline) and the card says 'within a day'. Five minutes is inside both by a wide margin and needs no hub change. It stops by construction — a healthy box does no work and logs nothing — and the settle gate is deliberately kept via ReconcileWhenSettled. The marker was investigated and left alone: applied_marker lives in the guest's DataDir, which a rebuild destroys, so it cannot suppress a legitimate re-run. R-220's customer half: the refusal no longer tells the customer to choose from a list that may be empty. It names the rebuild, points at the Meghajtók page, and promises no outcome. Red-proofs: remove the retry -> credential uncollected (the dead end reproduced); drop the stop condition -> a healthy box hammers the hub; call Reconcile instead of ReconcileWhenSettled -> settle gate bypassed; restore the old sentence -> the impossible action returns. 28 packages ok, vet clean, all controller gates OK.
2764 lines
128 KiB
Go
2764 lines
128 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"os/exec"
|
||
"os/signal"
|
||
"path/filepath"
|
||
"sort"
|
||
"syscall"
|
||
"time"
|
||
|
||
"crypto/subtle"
|
||
"strings"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/api"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/assets"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
|
||
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/mailrelay"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/offsiteapply"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/quiesce"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/recovery"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/selftest"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/selfupdate"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/setup"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||
catalogsync "gitea.dooplex.hu/admin/felhom-controller/internal/sync"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/web"
|
||
)
|
||
|
||
var (
|
||
// Set at build time via ldflags
|
||
Version = "dev"
|
||
BuildTime = "unknown"
|
||
GitCommit = "unknown"
|
||
)
|
||
|
||
func main() {
|
||
// Hidden re-exec mode (NAS verify-before-commit): the web server re-execs THIS binary as
|
||
// `felhom-controller --netprobe <dir>` with uid/gid-1000 credentials to prove a media app can
|
||
// write through a freshly-verified network share (SPIKE-nas-verify Q2). Handled before flag
|
||
// parsing; the exit code is the probe verdict (see web/netprobe.go).
|
||
if len(os.Args) == 3 && os.Args[1] == "--netprobe" {
|
||
os.Exit(web.NetProbeChild(os.Args[2]))
|
||
}
|
||
|
||
configPath := flag.String("config", "/opt/docker/felhom-controller/controller.yaml", "Path to configuration file")
|
||
showVersion := flag.Bool("version", false, "Show version and exit")
|
||
printResetCode := flag.Bool("print-reset-code", false, "Customer-claim escape hatch (v0.122.0, F-4): print a fresh one-time local claim/reset code to stdout, then exit. Root-gated by reachability (docker exec). Same gate consumes it.")
|
||
printInfraImages := flag.Bool("print-infra-images", false, "Print every controller-managed infra image (one per line) and exit. Read by the golden bake (felhom-agent configs/build-golden.sh) so the appliance image pre-pulls exactly what THIS controller version will request.")
|
||
recoverOffsiteCheck := flag.Bool("recover-offsite-check", false, "R-200 diagnostic: read the customer recovery code from STDIN, recover the offsite repository password from the hub-held sealed escrow via the agent, and report whether it matches the one on disk — BY HASH. Compares, never installs; writes nothing. Exit 0 = match, 2 = clean mismatch, 1 = a step failed.")
|
||
recoverOffsiteInstall := flag.Bool("recover-offsite-install", false, "R-200: read the customer recovery code from STDIN, recover the offsite repository password, print both hashes, and — with --confirm-install — PLACE it so the existing repository reopens. Without --confirm-install it is a dry run that writes nothing.")
|
||
confirmInstall := flag.Bool("confirm-install", false, "Required alongside --recover-offsite-install to actually write the recovered repository password. Deliberately a second invocation so the hashes are seen before any write is possible.")
|
||
flag.Parse()
|
||
|
||
if *showVersion {
|
||
fmt.Printf("felhom-controller %s (built %s, commit %s)\n", Version, BuildTime, GitCommit)
|
||
os.Exit(0)
|
||
}
|
||
|
||
// Config-free by design: the golden bake runs this against a bare `docker run` of the controller
|
||
// image, where no controller.yaml, data dir or settings exist yet. It must never touch either.
|
||
if *printInfraImages {
|
||
for _, img := range infra.Images() {
|
||
fmt.Println(img)
|
||
}
|
||
os.Exit(0)
|
||
}
|
||
|
||
// R-200 (v0.196.0) — the INSTALL sibling. Same fetch/unseal path, same stdin discipline for R;
|
||
// the difference is that it places the recovered password so a rebuilt box reopens the off-site
|
||
// history it inherited. Two invocations by design: without --confirm-install it prints the hashes
|
||
// and writes nothing.
|
||
if *recoverOffsiteInstall {
|
||
cfg, err := config.LoadPermissive(*configPath)
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-install: loading config: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", log.New(os.Stderr, "", 0))
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-install: loading settings: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-install: agent channel: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
os.Exit(backup.RecoverAndInstall(backup.RecoveryCheckDeps{
|
||
Manager: backup.NewManager(cfg, sett, log.New(os.Stderr, "", 0)),
|
||
Recoverer: ac,
|
||
In: os.Stdin,
|
||
Out: os.Stdout,
|
||
Err: os.Stderr,
|
||
}, *confirmInstall))
|
||
}
|
||
|
||
// R-200 (v0.195.0) — the offsite-key recovery check. A `docker exec` escape hatch in the shape of
|
||
// --print-reset-code above, and deliberately NOT a page or a browser-reachable API: the
|
||
// customer-facing flow is designed on top of a chain that has been walked, and this is the walk.
|
||
// R arrives on STDIN so it never appears in argv, `ps`, shell history or a session transcript.
|
||
if *recoverOffsiteCheck {
|
||
cfg, err := config.LoadPermissive(*configPath)
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-check: loading config: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", log.New(os.Stderr, "", 0))
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-check: loading settings: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "recover-offsite-check: agent channel: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
os.Exit(backup.RunRecoveryCheck(backup.RecoveryCheckDeps{
|
||
Manager: backup.NewManager(cfg, sett, log.New(os.Stderr, "", 0)),
|
||
Recoverer: ac,
|
||
In: os.Stdin,
|
||
Out: os.Stdout,
|
||
Err: os.Stderr,
|
||
}))
|
||
}
|
||
|
||
if *printResetCode {
|
||
cfg, err := config.LoadPermissive(*configPath)
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "print-reset-code: loading config: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", log.New(os.Stderr, "", 0))
|
||
if err != nil {
|
||
fmt.Fprintf(os.Stderr, "print-reset-code: loading settings: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
os.Exit(web.PrintLocalResetCode(sett, cfg))
|
||
}
|
||
|
||
startTime := time.Now()
|
||
|
||
// --- Load configuration ---
|
||
// Use LoadPermissive to tolerate minimal configs (e.g. only domain set by docker-setup.sh).
|
||
// If even that fails (file missing/unreadable), fall back to defaults.
|
||
cfg, err := config.LoadPermissive(*configPath)
|
||
if err != nil {
|
||
cfg = config.Default()
|
||
log.Printf("[WARN] Config load failed (%s), using defaults: %v", *configPath, err)
|
||
}
|
||
|
||
logger, logBuffer := setupLogger(cfg)
|
||
// fix-6 (CAMPAIGN-3): load the pre-restart debug-ring window back immediately, so a controller
|
||
// restart / container recreation no longer wipes the exact evidence an operator needs. The spill
|
||
// lives on the SSD state dir (DataDir — holds settings.json, survives recreation), NEVER a NAS path.
|
||
ringSpillPath := filepath.Join(cfg.Paths.DataDir, "debug-ring.log")
|
||
logBuffer.LoadFrom(ringSpillPath)
|
||
// v0.116.0: the debug ring is the controller_log_tail source (report self-tail channel).
|
||
report.SetControllerLogSource(logBuffer.Lines)
|
||
|
||
// --- Bootstrap ingestion (slice 8A → v0.40.0 onboarding, doc 03 §6) ---
|
||
// On first run, if this controller is not yet configured AND the host agent's provisioning
|
||
// back-half attached a bootstrap.json config mount, PULL the full controller.yaml from the hub
|
||
// (using the bootstrap's retrieval passphrase), merge in the per-guest local_api block, and come
|
||
// up CONFIGURED — skipping setup mode. Idempotent (never clobbers an existing controller.yaml)
|
||
// and fail-safe (a malformed/absent bootstrap, or a hub outage at first boot, leaves us in setup
|
||
// mode). The adapter marks a transient hub-unreachable error as retryable (the rest are permanent).
|
||
pull := func(hubURL, customerID, retrievalPassword string) (string, error) {
|
||
y, perr := report.PullConfig(hubURL, customerID, retrievalPassword)
|
||
if perr != nil && errors.Is(perr, report.ErrHubUnreachable) {
|
||
return "", fmt.Errorf("%w: %w", bootstrap.ErrPullTransient, perr)
|
||
}
|
||
return y, perr
|
||
}
|
||
cfg = bootstrap.MaybeIngest(*configPath, cfg, logger, pull)
|
||
|
||
// --- Wire system package debug logging ---
|
||
if cfg.Logging.Level == "debug" {
|
||
system.DebugLogger = logger
|
||
}
|
||
|
||
// --- Setup mode: if no customer ID configured, run setup wizard ---
|
||
if setup.NeedsSetup(cfg) {
|
||
logger.Printf("[INFO] felhom-controller %s — setup mode", Version)
|
||
runSetupMode(cfg, logger)
|
||
return
|
||
}
|
||
|
||
// --- Local API connectivity probe (slice 8A) ---
|
||
// When seeded with a local-API endpoint, prove the controller↔agent channel at startup and
|
||
// learn this guest's mounts (placement view). Non-fatal — the controller runs regardless; a
|
||
// failure is logged for diagnosis. The full /backup/due quiesce loop lands in 8B.
|
||
probeLocalAPI(cfg, logger)
|
||
|
||
logger.Printf("[INFO] felhom-controller %s starting (customer: %s, domain: %s)",
|
||
Version, cfg.Customer.ID, cfg.Customer.Domain)
|
||
|
||
// --- Load settings ---
|
||
settingsPath := cfg.Paths.DataDir + "/settings.json"
|
||
sett, err := settings.Load(settingsPath, logger)
|
||
if err != nil {
|
||
logger.Fatalf("[FATAL] Failed to load settings from %s: %v", settingsPath, err)
|
||
}
|
||
sett.SetDebug(cfg.Logging.Level == "debug")
|
||
|
||
// --- Auto-discover storage paths from deployed apps ---
|
||
discoveredPaths := discoverHDDPaths(cfg.Paths.StacksDir, logger)
|
||
sett.AutoDiscoverStoragePaths(discoveredPaths, cfg.Paths.HDDPath, logger)
|
||
|
||
// --- Load or create encryption key ---
|
||
encKeyPath := filepath.Join(cfg.Paths.DataDir, "encryption.key")
|
||
encKey, err := crypto.LoadOrCreateKey(encKeyPath)
|
||
if err != nil {
|
||
logger.Fatalf("[FATAL] Failed to load encryption key: %v", err)
|
||
}
|
||
logger.Printf("[INFO] Encryption key loaded from %s", encKeyPath)
|
||
|
||
// --- Initialize stack manager ---
|
||
stackMgr, err := stacks.NewManager(cfg, logger)
|
||
if err != nil {
|
||
logger.Fatalf("[FATAL] Failed to initialize stack manager: %v", err)
|
||
}
|
||
stackMgr.SetEncryptionKey(encKey)
|
||
|
||
// Initial stack scan
|
||
if err := stackMgr.ScanStacks(); err != nil {
|
||
logger.Printf("[WARN] Initial stack scan failed: %v", err)
|
||
}
|
||
|
||
// Inject missing deploy fields for all deployed stacks on startup
|
||
if names := stackMgr.DeployedStackNames(); len(names) > 0 {
|
||
stackMgr.InjectMissingFields(names)
|
||
}
|
||
|
||
// Migrate existing plaintext passwords to encrypted
|
||
stackMgr.MigrateEncryption()
|
||
|
||
// --- First-boot base-infrastructure bring-up ---
|
||
// We are guaranteed configured here (setup.NeedsSetup returned false above), so deploy the base
|
||
// stack (traefik-public network → traefik → cloudflared → filebrowser) the controller needs for
|
||
// routing + external access. Runs in a goroutine so a slow first-boot image pull never delays the
|
||
// web server; non-fatal (idempotent + single-flight, the health loop re-attempts each tick).
|
||
go func() {
|
||
if err := stackMgr.EnsureBaseStack(); err != nil {
|
||
logger.Printf("[WARN] [infra] first-boot base-stack bring-up: %v", err)
|
||
}
|
||
}()
|
||
|
||
// --- Initialize catalog syncer ---
|
||
syncer := catalogsync.New(cfg, logger, stackMgr.ScanStacks, func(updated []string) {
|
||
stackMgr.InjectMissingFields(updated)
|
||
})
|
||
syncer.Start()
|
||
defer syncer.Stop()
|
||
|
||
// --- Graceful shutdown context ---
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
|
||
// --- Quiesce loop (slice 8B): app-consistent backup around the agent vzdump ---
|
||
// Runs only when the local API is configured (a provisioned guest) and quiesce is enabled.
|
||
// Recover FIRST (restart any stacks left stopped by a crash mid-quiesce), then start the loop.
|
||
quiesceLoop := startQuiesceLoop(ctx, cfg, sett, stackMgr, logger)
|
||
|
||
// --- R-166: recover apps left stopped by an interrupted app-data operation ---
|
||
// A volume dump, an offsite reconstitution or a `.fab` export stops the app, works on its data,
|
||
// and starts it again. A controller killed inside that window used to leave the app down with
|
||
// NOTHING on disk explaining it — and a stopped app has zero containers, which the boot
|
||
// reconciler below read as a deliberate customer stop and left alone, indefinitely.
|
||
//
|
||
// ORDERING IS LOAD-BEARING (§8.4) and this call must COMPLETE, not merely be reached, before the
|
||
// boot-reconcile goroutine is launched: an app the marker already explains must not also be
|
||
// reported as an unexplained boot orphan. Same position and same reason as the quiesce Recover
|
||
// immediately above.
|
||
// The guard is built HERE rather than taken from the backup manager because that manager is not
|
||
// constructed until ~40 lines below — and moving its construction up to suit this would be a far
|
||
// wider change than moving one object down. It is handed to the manager (SetAppStopGuard) and to
|
||
// the exporter later, so all three share ONE guard over ONE file.
|
||
//
|
||
// R-174: the starter is GATED, never the raw manager. Recover runs here, at startup — exactly
|
||
// when an external drive may not have come back — and `Manager.StartStack` has no drive gate of
|
||
// its own, so the un-gated version started apps onto missing drives. The gate is the SAME
|
||
// `driveStartGate` the boot sweep uses (holder #3 of bootDriveGate), so the two cannot disagree.
|
||
appStopGuard := backup.NewAppStopGuard(filepath.Join(cfg.Paths.DataDir, "appstop-state.json"), logger)
|
||
appStopGuard.SetStarter(gatedAppStopStarter{
|
||
inner: stackMgr,
|
||
gate: driveStartGate{mgr: stackMgr, sett: sett},
|
||
logger: logger,
|
||
})
|
||
appStopRecovery := appStopGuard.Recover()
|
||
|
||
// --- R-166: desired-state backfill (running-only) ---
|
||
// Converge the apps whose intent is unambiguous — deployed and observed UP — so the fleet stops
|
||
// depending on legacy inference without waiting for a button press. NEVER backfills "stopped":
|
||
// zero containers cannot distinguish a deliberate stop from a power cut, and that inference is
|
||
// the defect. Runs after the two recoveries so a just-restarted app is counted as running.
|
||
stackMgr.BackfillDesiredState()
|
||
|
||
// --- R-52: boot desired-state reconciliation ---
|
||
// A deployed app that missed its boot start used to stay down until a human noticed (F5: immich
|
||
// and calibre-web sat Exited for ~18 h while ten siblings came back). One bounded start-once
|
||
// sweep, deliberately AFTER the quiesce recovery above so the two never race for the same stack,
|
||
// and entirely inside deadAppBootGrace so a successful recovery is silent and a failed one still
|
||
// alerts honestly. Never touches an app the customer stopped — see internal/bootrecon.
|
||
// R-171: hand the boot sweep the settings it needs to answer "is this app's drive live?" BEFORE
|
||
// the goroutine starts — an unwired gate is silently the pre-v0.190.0 behaviour that started apps
|
||
// onto absent drives. TestMainWiresBootDriveGate walks this file's AST for the assignment.
|
||
bootDriveSettings = sett
|
||
bootQuiesceLoop = quiesceLoop
|
||
bootAppStopGuard = appStopGuard
|
||
go runBootReconcile(ctx, stackMgr, logger)
|
||
|
||
// --- Start CPU collector ---
|
||
cpuCollector := system.NewCPUCollector(5 * time.Second)
|
||
cpuCollector.Start(ctx)
|
||
defer cpuCollector.Stop()
|
||
|
||
// --- Initialize metrics store + collector ---
|
||
metricsDBPath := "/opt/docker/felhom-controller/data/metrics.db"
|
||
metricsStore, err := metrics.NewMetricsStore(metricsDBPath, logger)
|
||
if err != nil {
|
||
logger.Printf("[WARN] Failed to initialize metrics store: %v — monitoring disabled", err)
|
||
} else {
|
||
logger.Printf("[INFO] Metrics store opened at %s", metricsDBPath)
|
||
}
|
||
|
||
if metricsStore != nil {
|
||
defer metricsStore.Close()
|
||
metricsHDDPath := cfg.Paths.HDDPath
|
||
if p := sett.GetDefaultStoragePath(); p != "" {
|
||
metricsHDDPath = p
|
||
}
|
||
metricsCollector := metrics.NewMetricsCollector(metricsStore, cpuCollector, metricsHDDPath, logger)
|
||
metricsCollector.Start(ctx)
|
||
defer metricsCollector.Stop()
|
||
logger.Println("[INFO] Metrics collector started (60s interval)")
|
||
}
|
||
|
||
// Deprecation notice for ping UUIDs (Healthchecks pinging retired — the Hub
|
||
// now owns monitoring; disk-tier backup moved to the host agent in slice 8C).
|
||
uuids := cfg.Monitoring.PingUUIDs
|
||
if uuids.Heartbeat != "" || uuids.SystemHealth != "" || uuids.DBDump != "" || uuids.Backup != "" || uuids.BackupIntegrity != "" {
|
||
logger.Println("[INFO] Healthchecks ping UUIDs configured but no longer used — monitoring is now handled by the Hub")
|
||
}
|
||
|
||
// --- Initialize backup manager (app-data only: DB dumps + Docker-volume tars) ---
|
||
var backupMgr *backup.Manager
|
||
stackProv := &stackAdapter{
|
||
mgr: stackMgr,
|
||
getStoragePaths: func() []settings.StoragePath { return sett.GetStoragePaths() },
|
||
encKey: encKey,
|
||
}
|
||
if cfg.Backup.Enabled {
|
||
backupMgr = backup.NewManager(cfg, sett, logger)
|
||
// R-166: use the guard that already ran Recover at startup, not a second one over the same
|
||
// file (see SetAppStopGuard — one file, one owner).
|
||
backupMgr.SetAppStopGuard(appStopGuard)
|
||
backupMgr.SetStackProvider(stackProv)
|
||
backupMgr.SetVersion(Version)
|
||
// O4: restore-from-unit generates a replacement for an unrecoverable RESETTABLE secret
|
||
// (data-keys stay fail-closed) so the app redeploys with a fresh credential, not a blank one.
|
||
backupMgr.SetSecretGenerator(stackMgr.GenerateSecretForField)
|
||
// R-7b: after a shares restore re-adds definitions to the registry, smb.conf must be
|
||
// re-rendered or the restored shares exist on paper but are not exported. A seam rather than a
|
||
// direct call — the backup package must not depend on the stacks package.
|
||
backupMgr.SetSharesReconciler(stackMgr.ReconcileSamba)
|
||
}
|
||
|
||
// SLICE 2: the offsite apply-bridge is launched further down, AFTER the self-updater is constructed
|
||
// (R-71a: the bridge's settle-gate reads the updater's floor/update-running state to defer the
|
||
// consume past a managed day-0 floor-update). See "offsite apply-bridge" below.
|
||
|
||
// --- Wire the data-migration engine (B1) + backup↔migration mutual exclusion (Change 3) ---
|
||
stackMgr.SetMigrationDeps(sett, func() bool { return backupMgr != nil && backupMgr.IsRunning() })
|
||
if backupMgr != nil {
|
||
backupMgr.SetMigrationRunningCheck(stackMgr.IsMigrating)
|
||
}
|
||
// RecoverMigration is started AFTER the web server's done-hook is wired (below), so a resumed
|
||
// decommission-migration still finalizes the source decommission on completion.
|
||
|
||
// --- Initialize alert manager ---
|
||
alertMgr := web.NewAlertManager(logger)
|
||
|
||
// --- Initialize notifier ---
|
||
notifier := notify.New(cfg.Hub.URL, cfg.Hub.APIKey, cfg.Customer.ID, sett, logger, cfg.Logging.Level == "debug")
|
||
|
||
// R-97a: wire the quiesce loop's hub-event seam. It MUST happen here and not in
|
||
// startQuiesceLoop, because the notifier is constructed after it — and it must happen at all,
|
||
// or the seam is inert (the "built but never wired" trap, four recorded instances). Reachability
|
||
// is covered by TestQuiesceTierNotifierIsWired in this package.
|
||
if quiesceLoop != nil {
|
||
quiesceLoop.SetTierNotifier(quiesceTierNotifier{n: notifier})
|
||
}
|
||
|
||
// R-166 §2.4: report an interrupted app-data operation to the operator, HERE, because the
|
||
// recovery itself had to run before the boot reconciler (line ~236) and the notifier does not
|
||
// exist until this line. An interrupted operation means the controller died mid-backup and that
|
||
// backup did not complete — operator-grade news even when every app came back.
|
||
//
|
||
// It rides the EXISTING `backup_failed` event type rather than a new one: a new type needs the
|
||
// hub's allowedEventTypes + customerMessages pair changed, which is a wire change, and this
|
||
// release ships no hub change. A controller emitting an unlisted type gets a flat 400 from
|
||
// POST /event. Reachability is covered by TestAppStopRecoveryIsWired.
|
||
// R-174: `Alarming()` and not merely `!= nil`. A recovery that ONLY refused starts is the drive
|
||
// gate working as designed, and `backup_failed` is customer-enabled by default — reporting a
|
||
// deliberate hold through it would email the customer "A biztonsági mentés sikertelen!" about an
|
||
// app nothing is wrong with. The refusal is already logged at WARN with its reason.
|
||
if appStopRecovery != nil && appStopRecovery.Alarming() {
|
||
notifier.NotifyBackupFailed(appStopRecovery.Message(), appStopRecovery.Detail())
|
||
} else if appStopRecovery != nil {
|
||
logger.Printf("[WARN] [appstop] %s — not alarming: %s", appStopRecovery.Message(), appStopRecovery.Detail())
|
||
}
|
||
|
||
// --- Initialize the app-email SMTP shim (mailrelay) ---
|
||
// In-process shim: apps → shim → hub → Resend (the Resend key stays hub-side). It runs only
|
||
// when the controller has a hub (URL+key) AND the operational kill-switch is on; the runtime
|
||
// global app-email toggle then decides whether it actually listens (Lifecycle.Apply). Listeners
|
||
// bind to the app Docker network only — never published to the host/internet.
|
||
var mailShim *mailrelay.Lifecycle
|
||
if cfg.Hub.URL != "" && cfg.Hub.APIKey != "" && cfg.MailRelay.HardEnabled() {
|
||
mailShim = mailrelay.NewLifecycle(mailrelay.Options{
|
||
PlainAddr: cfg.MailRelay.PlainListen,
|
||
TLSAddr: cfg.MailRelay.TLSListen,
|
||
PlainNoTLSAddr: cfg.MailRelay.PlainNoTLSListen,
|
||
ServiceName: cfg.MailRelay.ShimHost,
|
||
Policy: mailrelay.NewPolicy(cfg.MailRelay.FromDomains),
|
||
Forwarder: mailrelay.NewHubForwarder(cfg.Hub.URL, cfg.Hub.APIKey),
|
||
Logger: logger,
|
||
})
|
||
if err := mailShim.Apply(sett.AppEmailEnabled()); err != nil {
|
||
logger.Printf("[ERROR] [mailrelay] could not start app-email shim: %v", err)
|
||
}
|
||
} else {
|
||
logger.Printf("[INFO] [mailrelay] app-email shim unavailable (no hub configured or disabled in config)")
|
||
}
|
||
|
||
// --- Initialize self-updater ---
|
||
var updater *selfupdate.Updater
|
||
if cfg.SelfUpdate.Enabled {
|
||
// Phase 1: the host agent performs the container swap. Build a (nil-able) agent client from the
|
||
// provisioned local-API config; when absent (un-provisioned guest) self-update is unavailable.
|
||
var swapAgent selfupdate.AgentSwapper
|
||
if cfg.LocalAPI.Endpoint != "" && cfg.LocalAPI.Token != "" {
|
||
if ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint); err != nil {
|
||
logger.Printf("[WARN] Self-update: agent client init failed (%v) — updates unavailable", err)
|
||
} else {
|
||
swapAgent = ac
|
||
}
|
||
}
|
||
updater = selfupdate.NewUpdater(&cfg.SelfUpdate, &cfg.Git, Version, cfg.Paths.DataDir, swapAgent, logger, cfg.Logging.Level == "debug")
|
||
updater.SetBackupRunningCheck(func() bool {
|
||
return backupMgr != nil && backupMgr.IsRunning()
|
||
})
|
||
// Check for post-update state (did a previous update succeed or fail?)
|
||
if state := updater.VerifyStartup(); state != nil {
|
||
notifier.NotifyControllerUpdated(state.PreviousVersion, state.TargetVersion, state.Status == "success")
|
||
}
|
||
logger.Printf("[INFO] Self-update enabled (check every %s, auto-update: %v, auto-update time: %s)",
|
||
cfg.SelfUpdate.CheckInterval, cfg.SelfUpdate.AutoUpdate, cfg.SelfUpdate.AutoUpdateTime)
|
||
}
|
||
|
||
// SLICE 2: the offsite apply-bridge — on startup (async, non-blocking) reconcile the hub-served
|
||
// offsite descriptor into a configured key-only offbox target (fail-safe, idempotent, no blind
|
||
// TOFU). The config_refresh self-restart re-runs this after a descriptor change (new process →
|
||
// startup). Launched HERE (after the self-updater is built) so the R-71a settle-gate can read the
|
||
// updater's floor/update-running state and defer the one-time-password consume past a managed
|
||
// day-0 floor-update (the F10 race). The gate is wired ONLY when an updater exists — with no update
|
||
// mechanism there is no floor-update to race, so the bridge reconciles immediately (Settle nil).
|
||
// offsiteBridge is hoisted out of the block below ONLY so the recovery screen can drive one
|
||
// reconcile between placing a recovered key and reading the repository (R-219). nil when off-site
|
||
// is not configured for this customer, which the web seam treats as "skip".
|
||
var offsiteBridge *offsiteapply.Bridge
|
||
if backupMgr != nil && cfg.Offsite.Enabled && cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
|
||
bridge := &offsiteapply.Bridge{
|
||
Cfg: cfg,
|
||
Consumer: offsiteapply.HTTPConsumer{HubURL: cfg.Hub.URL, CustomerID: cfg.Customer.ID, APIKey: cfg.Hub.APIKey},
|
||
Scanner: offsiteapply.KeyscanScanner{},
|
||
KeyGen: offsiteapply.ED25519KeyGen{},
|
||
Installer: offsiteapply.SSHCopyIDInstaller{},
|
||
Prober: offsiteapply.SFTPKeyAuthProber{KeyPath: filepath.Join(cfg.Paths.DataDir, "offbox", "ssh_key")},
|
||
Enabler: offsiteapply.EnablerFunc(func(ctx context.Context, host, user string, port int, repoPath, priv, kh string, quotaGB int) error {
|
||
tgt := &settings.OffboxTarget{Enabled: true, Host: host, User: user, Port: port, RepoPath: repoPath, Schedule: "daily", QuotaGB: quotaGB}
|
||
stage := func(ctx context.Context, pw string) error {
|
||
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return ac.StageEscrowSecret(ctx, pw)
|
||
}
|
||
return backupMgr.ApplyOffsiteTarget(ctx, tgt, priv, kh, stage)
|
||
}),
|
||
MarkerPath: filepath.Join(cfg.Paths.DataDir, "offbox", "applied_marker"),
|
||
Logger: logger,
|
||
}
|
||
if updater != nil {
|
||
// Thin adapter over the updater's OWN knowledge (StackDataProvider pattern) — the bridge
|
||
// never fetches the floor a second way. floorKnown = the floor has been learned from a
|
||
// report ACK yet (GetFloor() != "").
|
||
u := updater
|
||
bridge.Settle = offsiteapply.SettleFunc(func() (string, string, bool, bool) {
|
||
floor := u.GetFloor()
|
||
return Version, floor, u.IsUpdateRunning(), floor != ""
|
||
})
|
||
}
|
||
offsiteBridge = bridge
|
||
go func() {
|
||
// ReconcileWhenSettled runs the settle-gate FIRST (its own bounds), then Reconcile under a
|
||
// fresh 3-minute context — the gate's wait never eats the reconcile budget.
|
||
if err := bridge.ReconcileWhenSettled(context.Background()); err != nil {
|
||
logger.Printf("[WARN] [offsite-apply] reconcile: %v (retries on next config refresh/restart)", err)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// --- Initialize scheduler ---
|
||
sched := scheduler.New(logger)
|
||
sched.SetDebug(cfg.Logging.Level == "debug")
|
||
|
||
// Existing periodic tasks (migrated from ad-hoc goroutines)
|
||
// F7: 30s left the dashboard list lagging Docker health by up to ~30s after a deploy/state change.
|
||
// 10s (matching the health-probes cadence) tightens it; RefreshStatus is a cheap `docker ps`-based
|
||
// refresh of the in-memory map, so 10s does not meaningfully load Docker. (The deploy page itself
|
||
// already polls per-stack every 3s; this is for the dashboard/stacks list.)
|
||
sched.Every("status-refresh", 10*time.Second, func(ctx context.Context) error {
|
||
return stackMgr.RefreshStatus()
|
||
})
|
||
sched.Every("stack-scan", 2*time.Minute, func(ctx context.Context) error {
|
||
return stackMgr.ScanStacks()
|
||
})
|
||
sched.Every("health-probes", 10*time.Second, func(ctx context.Context) error {
|
||
return stackMgr.RunHealthProbes()
|
||
})
|
||
|
||
// System health check — refreshes dashboard alerts and notifies on changes.
|
||
// Healthchecks.io pinging has been retired (the Hub now owns monitoring).
|
||
healthInterval, err := time.ParseDuration(cfg.Monitoring.SystemHealthInterval)
|
||
if err != nil {
|
||
healthInterval = 5 * time.Minute
|
||
}
|
||
sched.Every("system-health", healthInterval, func(ctx context.Context) error {
|
||
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), sett.GetSMBSettings(), logger)
|
||
// Self-heal the base stack: call unconditionally every tick. EnsureBaseStack is single-flight
|
||
// + idempotent (skips running stacks ⇒ a cheap 3× docker-inspect no-op when healthy), so there
|
||
// is no need to couple to the health-report issue strings. Runs in a goroutine — never blocks
|
||
// or fails the health job.
|
||
go func() {
|
||
if err := stackMgr.EnsureBaseStack(); err != nil {
|
||
logger.Printf("[WARN] [infra] self-heal base-stack bring-up: %v", err)
|
||
}
|
||
}()
|
||
// Refresh dashboard alerts from health report
|
||
updateAvailable := false
|
||
latestVersion := ""
|
||
if updater != nil {
|
||
status := updater.GetStatus()
|
||
if status.LastCheck != nil {
|
||
updateAvailable = status.LastCheck.UpdateAvailable
|
||
latestVersion = status.LastCheck.LatestVersion
|
||
}
|
||
}
|
||
alertMgr.Refresh(healthReport, cfg, backupMgr, updateAvailable, latestVersion, sett.GetStoragePaths())
|
||
// Notify on health status changes
|
||
notifier.NotifyHealthChange(healthReport.Status, healthReport.Issues, healthReport.Warnings)
|
||
return nil
|
||
})
|
||
|
||
// fix-3 (CAMPAIGN-3): a deployed app that is not running must be LOUD, not silent (the campaign's
|
||
// CWA sat dead 4 h; F11 then produced 4 silently-dead NAS apps per reboot). Runs on its own short
|
||
// cadence (faster than the 5-min health cycle) so a dead app surfaces quickly. Logs nothing on the
|
||
// quiet path (no ring spam — fix-6 friendly); the dashboard banner is state-based (self-clears) and
|
||
// the hub event fires once per running→down transition (notifier tracks it). A boot grace skips the
|
||
// controller's own startup settle so apps that legitimately take 30–60 s to come up don't alert.
|
||
//
|
||
// F-OBS (Campaign 8): this check had NO positive observable at default `info` level. Its per-cycle
|
||
// scheduler line is emitted through Scheduler.dbg(), which is gated on logging.level==debug and so
|
||
// is never PRODUCED on a default box (not merely filtered — the always-DEBUG ring cannot capture
|
||
// it either), and a 30 s interval also puts it on the scheduler's quiet path. So on every customer
|
||
// box, "no alarms" was indistinguishable from "the detector never ran" — the precise fallacy this
|
||
// project now has a standing rule against, and it directly undermines confidence in the F-CRIT-1
|
||
// fix in the field.
|
||
//
|
||
// The observable is a PERIODIC SUMMARY, not a line per run: at 30 s a per-run line is 2880
|
||
// lines/day of pure noise, which is what made the original author choose silence. Every
|
||
// deadAppHeartbeatEvery-th scan emits one INFO carrying the scan count and what it found, so an
|
||
// operator can always answer "is it running, and what does it see?" from a default box — and a
|
||
// STALLED detector is visible as the heartbeat stopping.
|
||
sched.Every("deadapp-check", 30*time.Second, func(ctx context.Context) error {
|
||
if time.Since(startTime) < deadAppBootGrace {
|
||
return nil // still inside the startup settle window
|
||
}
|
||
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop)
|
||
alertMgr.SetDeadAppAlerts(dead)
|
||
notifier.NotifyAppStartFailures(states)
|
||
deadAppScans++
|
||
noteDeadAppScan(logger, deadAppScans, len(states), len(dead))
|
||
return nil
|
||
})
|
||
|
||
// fix-6 (CAMPAIGN-3): periodically spill the debug ring to the SSD state dir so a HARD crash loses
|
||
// at most ~60 s of the window (a clean shutdown spills too). ≤30s interval → auto-quiet (no
|
||
// per-cycle scheduler line). SpillTo is atomic (tmp+rename) so it can never corrupt the ring file.
|
||
sched.Every("ring-spill", 30*time.Second, func(ctx context.Context) error {
|
||
return logBuffer.SpillTo(ringSpillPath)
|
||
})
|
||
|
||
// --- Central hub pusher (declared early so backup closure can reference it) ---
|
||
var hubPusher *report.Pusher
|
||
// escrowConfirmer is hoisted so the web server (built later) can read its Scenario-F
|
||
// stale-blob flag (SetEscrowStale below). nil when no hub is configured.
|
||
var escrowConfirmer *report.EscrowAutoConfirmer
|
||
if cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
|
||
hubPusher = report.NewPusher(&cfg.Hub, logger, cfg.Logging.Level == "debug")
|
||
// SLICE 3 — hub-verified escrow auto-confirm (long-lived: the mismatch warn dedupes per hash,
|
||
// not per 15-min cycle). Flips offbox pending→escrowed ONLY when the hub-recorded hash of the
|
||
// escrowed password matches the local repo password's hash; never un-confirms. v0.127.0 adds
|
||
// the escrowed-state STALE re-check (Scenario F — warn + card flag, never a state change).
|
||
escrowConfirmer = &report.EscrowAutoConfirmer{
|
||
Pending: func() bool {
|
||
return backupMgr != nil && backupMgr.OffboxConfigured() &&
|
||
sett.GetOffboxTarget() != nil && sett.GetOffboxTarget().EscrowState == "pending"
|
||
},
|
||
Escrowed: func() bool {
|
||
return backupMgr != nil && backupMgr.OffboxConfigured() &&
|
||
sett.GetOffboxTarget() != nil && sett.GetOffboxTarget().EscrowState == "escrowed"
|
||
},
|
||
LocalHash: func() (string, bool) {
|
||
if backupMgr == nil {
|
||
return "", false
|
||
}
|
||
return backupMgr.OffboxRepoPasswordHash()
|
||
},
|
||
Flip: func() error {
|
||
return sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||
o.EscrowState = "escrowed"
|
||
o.CeremonyCompletedAt = "" // v0.138.0: clear the awaiting-card stamp on confirm
|
||
})
|
||
},
|
||
Wipe: func(ctx context.Context) error {
|
||
ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return ac.WipeStagedEscrowSecret(ctx)
|
||
},
|
||
// v0.199.0 (R-204 item 4 / R-193): remember whether the HUB holds a sealed recovery
|
||
// package for us — recorded on EVERY ACK, including when no off-site target exists, which
|
||
// is precisely the case that needs it. Half of the stranded-rebuild predicate in
|
||
// backup.needsOffsiteCredential; the other half (no repository password) is local.
|
||
RecordPresence: sett.SetHubEscrowIdentityPresent,
|
||
// v0.201.0 (R-222): whether the hub is ALSO keeping an EARLIER sealed package, so the
|
||
// recovery screen can name that situation instead of blaming the customer's typing.
|
||
RecordSuperseded: sett.SetHubEscrowSuperseded,
|
||
Logger: logger,
|
||
}
|
||
// Wire hub verification: update settings when hub reports customer status
|
||
hubPusher.OnPushResponse = func(resp *report.PushResponse) {
|
||
if resp.CustomerBlocked {
|
||
sett.SetHubVerified(false, time.Now())
|
||
logger.Printf("[WARN] Customer blocked on Hub — new deployments may be restricted")
|
||
} else {
|
||
sett.SetHubVerified(true, time.Now())
|
||
}
|
||
// Phase 2 managed updates: the ACK carries the operator-enforced minimum version (FLOOR).
|
||
// Hand it to the updater and reconcile — if the box is below the floor it auto-updates to
|
||
// the floor (reusing the Phase 1 swap). This rides the existing report cycle; no new timer.
|
||
// latest_version stays informational (the customer's opt-in button), NOT the auto-target.
|
||
if updater != nil {
|
||
updater.SetFloor(resp.MinControllerVersion)
|
||
updater.MaybeAutoUpdate()
|
||
}
|
||
// Config-refresh (v0.26.0): the ACK also carries the per-customer config_version. On a
|
||
// change vs. the last-applied version, re-pull controller.yaml (re-merging local_api from
|
||
// bootstrap.json) and self-restart so the new config loads. Pull-based — the hub never
|
||
// connects into the box. Rides this same report cycle; no new timer, no agent. First-ever
|
||
// ACK records the baseline without restarting; a failed pull keeps the current config and
|
||
// retries next cycle; an unchanged version is a no-op (so no restart storm).
|
||
cr := &report.ConfigRefresher{
|
||
Applied: sett.GetAppliedConfigVersion,
|
||
Record: sett.SetAppliedConfigVersion,
|
||
Refresh: func() error { return bootstrap.RefreshConfig(*configPath, logger, pull) },
|
||
Restart: func() { api.GracefulSelfRestart(logger) },
|
||
Logger: logger,
|
||
}
|
||
cr.Reconcile(resp.ConfigVersion)
|
||
// SLICE 3: run the escrow auto-confirm on the same ACK (after the config refresh decision —
|
||
// a refresh-restart re-enters here anyway on the next cycle).
|
||
escrowConfirmer.Reconcile(resp.Escrow)
|
||
// v0.111.0: operator log-tail requests ride the same ACK (pull pattern — the hub
|
||
// never reaches in). The NEXT report cycle collects + ships the tails; the hub
|
||
// clears its pending request on receipt. An empty list clears any stale local set.
|
||
report.SetPendingLogTails(resp.LogTailRequests)
|
||
// v0.116.0: the controller's OWN ring, same pattern (selftail.go).
|
||
report.SetPendingControllerLog(resp.ControllerLogRequested)
|
||
// v0.122.0 (F-4): cache the hub-delivered claim-code state (idempotent by
|
||
// generation). The web gate reads it on the next request — no restart needed.
|
||
claimSync := &report.ClaimSync{Settings: sett, Logger: logger}
|
||
claimSync.Reconcile(resp.Claim)
|
||
}
|
||
// Wire hub push status into alert manager for dashboard alerts
|
||
alertMgr.SetHubPushStatus(func() web.HubPushStatusData {
|
||
s := hubPusher.GetStatus()
|
||
return web.HubPushStatusData{
|
||
LastAttempt: s.LastAttempt,
|
||
LastSuccess: s.LastSuccess,
|
||
LastError: s.LastError,
|
||
Consecutive: s.Consecutive,
|
||
}
|
||
})
|
||
}
|
||
|
||
// Backup daily jobs
|
||
if cfg.Backup.Enabled && backupMgr != nil {
|
||
// v0.168.0: ONE customer setting (the window start W) drives all three nightly legs at fixed
|
||
// offsets — db-dump at W, tier-2 at W+60m, off-box at W+105m — so they can never be misordered.
|
||
// The window resolves settings > controller.yaml > "02:30"; a UI save fans out via UpdateDaily.
|
||
win := backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
|
||
dbLeg, tier2Leg, offboxLeg := backupwindow.LegTimes(win)
|
||
|
||
// App-data backup: daily database dumps. Disk-tier (restic snapshots,
|
||
// cross-drive, integrity check, infra backup) has moved to the host agent.
|
||
sched.Daily("db-dump", dbLeg, func(ctx context.Context) error {
|
||
err := backupMgr.RunDBDumps(ctx)
|
||
if err != nil {
|
||
notifier.NotifyDBDumpFailed("Adatbázis mentés sikertelen", err.Error())
|
||
} else {
|
||
notifier.NotifyDBDumpCompleted(notify.DBDumpDetails{})
|
||
}
|
||
return err
|
||
})
|
||
|
||
// ── R-218, THE CONSUME HALF (v0.203.0) ────────────────────────────────────────────────
|
||
//
|
||
// The declaration half shipped in v0.201.0 and works: a stranded box says
|
||
// `offsite.state=needs_credential`, and the hub's `offsiteheal` re-stages the one-time secret
|
||
// and logs *"the box re-consumes on its next cycle"*.
|
||
//
|
||
// **THERE WAS NO NEXT CYCLE.** `Reconcile` ran exactly twice in a process's life: once at
|
||
// start-up (the goroutine above, whose own comment says "retries on next config refresh/
|
||
// restart") and once when the recovery screen drives it (R-219). Both fire BEFORE the hub has
|
||
// anything staged, because the hub only stages in response to the declaration those runs
|
||
// precede.
|
||
//
|
||
// Measured on the R-201 re-walk, 2026-08-06: unlock reconcile 11:43:07 · hub staged 11:44:57
|
||
// saying "next cycle" · a full report cycle ran 11:55:46 · still unconsumed at 12:06. A guest
|
||
// command line moved it in **18 seconds**, which proves the credential, the target and the key
|
||
// were all fine and only the trigger was missing.
|
||
//
|
||
// So: re-run the SAME reconcile, on a tick, for exactly as long as the box itself says it needs
|
||
// a credential. The signal is the box's own `OffboxReportStatus().State` — the very statement
|
||
// the hub acts on, so the two can never disagree about whether a retry is wanted.
|
||
//
|
||
// WHY A POLL AND NOT AN ACK FLAG: the customer-facing text promises "amint megvannak" (no
|
||
// deadline) and the backups card promises "within a day"; a 5-minute tick is inside both by a
|
||
// wide margin, and it needs no hub change. If either promise ever tightens to minutes, revisit.
|
||
//
|
||
// IT STOPS BY CONSTRUCTION: the instant a target exists `OffboxReportStatus` stops returning
|
||
// the declared state, so a healthy box does no work and makes no noise. The settle gate is
|
||
// deliberately kept — `ReconcileWhenSettled` waits for floor knowledge exactly as at start-up.
|
||
sched.Every("offsite-credential-retry", 5*time.Minute, func(ctx context.Context) error {
|
||
if offsiteBridge == nil {
|
||
return nil // off-site not configured for this customer
|
||
}
|
||
attempted, err := offsiteBridge.RetryIfDeclared(ctx, func() bool {
|
||
st := backupMgr.OffboxReportStatus()
|
||
return st != nil && st.State == backup.OffsiteStateNeedsCredential
|
||
})
|
||
if !attempted {
|
||
return nil // a target exists (or we never needed one) — no work, and no log line
|
||
}
|
||
if err != nil {
|
||
// NOT a job failure: the box still declares, so the next tick tries again. Logged at
|
||
// WARN because a credential that never arrives is exactly what this exists to surface.
|
||
logger.Printf("[WARN] [offsite-apply] credential retry: %v (the box still declares a need; retrying)", err)
|
||
return nil
|
||
}
|
||
logger.Printf("[INFO] [offsite-apply] credential retry: the staged credential was collected and the tier applied")
|
||
return nil
|
||
})
|
||
|
||
// Cache refresh: every 5 minutes. Recompute the effective window each pass so the cached
|
||
// "next DB dump" follows a runtime window change (the UI save also refreshes immediately).
|
||
sched.Every("backup-cache", 5*time.Minute, func(ctx context.Context) error {
|
||
curDB, _, _ := backupwindow.LegTimes(backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule))
|
||
backupMgr.RefreshCache(scheduler.NextDailyRun(curDB))
|
||
return nil
|
||
})
|
||
|
||
// Tier 2: off-drive copy of each HDD app's recovery unit + userdata (auto-enabled, auto-target).
|
||
// Runs after the DB dump so it copies a fresh unit.
|
||
backupMgr.SetTier2Notifier(func(stackName, destLabel string, dur time.Duration, err error) {
|
||
// DISPLAY BOUNDARY (R-7b): this event's StackName lands in Hungarian operator/customer copy
|
||
// ("Másodlagos mentés elkészült: <név>"), so the reserved shares key is mapped here. Caught
|
||
// by live validation — the first demo run pushed a literal "_shares" before this line.
|
||
display := backup.DisplayStackName(stackName)
|
||
if err != nil {
|
||
notifier.NotifyCrossDriveFailed(notify.CrossDriveDetails{
|
||
StackName: display, Method: "rsync", DestPath: destLabel,
|
||
Duration: dur.Round(time.Second).String(), Error: err.Error(),
|
||
})
|
||
} else {
|
||
notifier.NotifyCrossDriveCompleted(notify.CrossDriveDetails{
|
||
StackName: display, Method: "rsync", DestPath: destLabel,
|
||
Duration: dur.Round(time.Second).String(),
|
||
})
|
||
}
|
||
})
|
||
sched.Daily("tier2-backup", tier2Leg, func(ctx context.Context) error {
|
||
backupMgr.RunAllTier2()
|
||
return nil
|
||
})
|
||
|
||
// Off-box (NAS) restic-SFTP backup (Part B): the off-site leg. A failure (incl. a fail-fast dead-NAS
|
||
// error) alerts the operator via the allowlisted backup_failed event. Daily after Tier 2.
|
||
backupMgr.SetOffboxNotify(func(dur time.Duration, snapshots int, err error) {
|
||
if err != nil {
|
||
// F-DIAG: a distinct message per cause, and the detail SANITISED — a raw err.Error()
|
||
// carries the sftp:user@host:/path repo reference off the box.
|
||
notifier.NotifyBackupFailed("Off-box (NAS) mentés sikertelen",
|
||
backupMgr.OffsiteFailureMessage(err, dur))
|
||
}
|
||
})
|
||
// R-203 (operator half): a completed off-site run could NOT capture a directory an app declares
|
||
// MANDATORY. It is not a failed run — what was captured is real and the snapshot count records
|
||
// it — but the app is NOT fully protected, and until v0.197.0 that reached nobody: the run said
|
||
// ok, the card said ok, the hub said ok, and one WARN line inside the container said otherwise.
|
||
//
|
||
// It rides the EXISTING per-run digest (backup_run_failures) rather than a new event type: a
|
||
// new type is a two-repo change (the hub's allowedEventTypes drops anything unlisted), and the
|
||
// digest is already operator-only and already means "this run did not fully do its job".
|
||
backupMgr.SetOffboxGapNotify(func(gaps map[string][]string) {
|
||
apps := make([]string, 0, len(gaps))
|
||
for app := range gaps {
|
||
apps = append(apps, app)
|
||
}
|
||
sort.Strings(apps) // deterministic message + details
|
||
d := notify.BackupRunFailuresDetails{RunKind: "offsite", Attempted: len(apps), Failed: len(apps)}
|
||
var parts []string
|
||
for _, app := range apps {
|
||
sort.Strings(gaps[app])
|
||
reason := "declared MANDATORY data directories not captured: " + strings.Join(gaps[app], ", ")
|
||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: app, Leg: "offsite-userdata", Reason: reason})
|
||
parts = append(parts, fmt.Sprintf("%s (%s)", app, strings.Join(gaps[app], ", ")))
|
||
}
|
||
notifier.NotifyBackupRunFailures(fmt.Sprintf(
|
||
"Offsite backup INCOMPLETE: %d app(s) had a directory they declare MANDATORY missing from the snapshot — %s. "+
|
||
"The run itself succeeded and what it captured is real, but these apps are NOT fully protected off-site. "+
|
||
"Check that the declared path exists on disk and that the app's data root resolves where the backup looks (R-203).",
|
||
len(apps), strings.Join(parts, "; ")), d)
|
||
})
|
||
// R-158 / R-167 (D-c, operator half): a per-app Tier-1 recovery-unit capture failed. Until
|
||
// v0.191.0 this was a `[WARN]` line and nothing else — /backups/apps is the page you open to
|
||
// ask whether ONE app is backed up, and it was the one page that never said. Fires per app;
|
||
// the capture loop continues, so three failing apps produce three events and one failing app
|
||
// does not silence its siblings.
|
||
//
|
||
// OPERATOR-TIER, not backup_failed — a customer can take no action on a capture failure, and
|
||
// backup_failed is customer-enabled by default. The space figures ride along because the
|
||
// overwhelmingly likely cause is a full filesystem and they answer "why" without a login.
|
||
// No controller-side cooldown: the hub owns it.
|
||
backupMgr.SetUnitNotify(func(stackName string, err error, usage *backup.UnitSpace) {
|
||
d := notify.RecoveryUnitFailureDetails{App: stackName, Error: err.Error()}
|
||
if usage != nil {
|
||
d.TargetPath, d.UsedGB, d.AvailGB = usage.Path, usage.UsedGB, usage.AvailGB
|
||
d.TotalGB, d.UsedPercent, d.SpaceKnown = usage.TotalGB, usage.UsedPercent, true
|
||
}
|
||
notifier.NotifyRecoveryUnitCaptureFailed(fmt.Sprintf(
|
||
"Recovery unit capture FAILED for %q — the app has no fresh local (Tier-1) backup, and Tier-2/Tier-3 have nothing to copy. %s. Error: %v",
|
||
stackName, usage.String(), err), d)
|
||
})
|
||
// R-182: the ONE operator digest per backup run. The per-app events above are the RECORD
|
||
// (the hub routes them record-only); this is the NOTIFICATION, sent once at the end of a run
|
||
// and only when something actually failed. A clean run sends nothing — and that silence is
|
||
// safe because the hub's own daily deadline check raises expected_backup_missed from report
|
||
// freshness, independently of any mail this box chooses to send.
|
||
backupMgr.SetRunSummaryNotify(func(rs backup.RunSummary) {
|
||
d := notify.BackupRunFailuresDetails{
|
||
RunID: rs.RunID, RunKind: rs.RunKind,
|
||
Failed: rs.Failed, Attempted: rs.Attempted,
|
||
}
|
||
for _, a := range rs.Apps {
|
||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: a.App, Leg: a.Leg, Reason: a.Reason})
|
||
}
|
||
if rs.Usage != nil {
|
||
d.TargetPath, d.UsedGB, d.AvailGB = rs.Usage.Path, rs.Usage.UsedGB, rs.Usage.AvailGB
|
||
d.TotalGB, d.UsedPercent, d.SpaceKnown = rs.Usage.TotalGB, rs.Usage.UsedPercent, true
|
||
}
|
||
notifier.NotifyBackupRunFailures(rs.Message, d)
|
||
})
|
||
// 3a: the pre-push enlargement gate blocked an app's userdata push (config+DB still saved). Edge-
|
||
// triggered by the engine (only NEW blocks notify), so the hub's per-event-type cooldown suffices —
|
||
// no controller-side timer (the hub owns cooldown).
|
||
backupMgr.SetOffboxEnlargeBlockedNotifier(func(stack string, estBytes int64, usedGB, quotaGB int) {
|
||
notifier.NotifyOffboxEnlargeBlocked(fmt.Sprintf(
|
||
"A(z) %s teljes távoli mentése (~%s) túllépné a tárhelykeretet (%d/%d GB). A konfiguráció és az adatbázis továbbra is mentésre kerül; nagyobb kerethez vedd fel velünk a kapcsolatot.",
|
||
stack, appbackup.HumanizeBytes(estBytes), usedGB, quotaGB))
|
||
})
|
||
// v0.142.0 offsite-repo continuity: push a hub event on the orphaned/reset transitions (once per
|
||
// transition — the Manager guards nightly re-fire). Operator-visible on the customer page.
|
||
backupMgr.SetOffboxOrphanEvent(func(eventType, renamedTo string) {
|
||
switch eventType {
|
||
case "offbox_repo_orphaned":
|
||
notifier.PushEvent("offbox_repo_orphaned", "warning",
|
||
"A távoli mentési tároló elárvult: a benne lévő mentések egy korábbi, már nem elérhető kulccsal készültek (újratelepítés). Új mentés a tároló visszaállításáig nem készül.", nil)
|
||
case "offbox_repo_reset":
|
||
notifier.PushEvent("offbox_repo_reset", "info",
|
||
"A távoli mentési tároló visszaállítva: a régi előzmény félretéve (nem törölve), és egy üres, új tároló jött létre a mostani kulccsal.", map[string]string{"renamed_to": renamedTo})
|
||
}
|
||
})
|
||
sched.Daily("offbox-backup", offboxLeg, func(ctx context.Context) error {
|
||
t := sett.GetOffboxTarget()
|
||
if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
|
||
return nil // not configured / not scheduled
|
||
}
|
||
return backupMgr.RunOffboxBackup(ctx)
|
||
})
|
||
}
|
||
|
||
// Metrics prune — daily at 04:00
|
||
if metricsStore != nil {
|
||
sched.Daily("metrics-prune", "04:00", func(ctx context.Context) error {
|
||
deleted, err := metricsStore.Prune(30 * 24 * time.Hour)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
logger.Printf("[INFO] Pruned %d old metric rows", deleted)
|
||
return nil
|
||
})
|
||
}
|
||
|
||
// --- R-167 (decision D-c, customer half): warn BEFORE a filesystem fills ---
|
||
//
|
||
// Nothing warned before v0.191.0. The first sign of a full filesystem was a backup that did not
|
||
// happen, and the only related signal — the healthcheck's generic `health_degraded` at 90% — looked
|
||
// at REGISTERED STORAGE PATHS ONLY, so the docker area and the system-data area (the one holding
|
||
// every driveless app's recovery unit) were invisible, and it never reported a free-byte figure or
|
||
// named a drive.
|
||
//
|
||
// CADENCE: DAILY, at 03:30. A fill is a slow-moving quantity — the thing that fills a disk is a
|
||
// customer's photo library or a nightly backup, not a spike — so a shorter interval buys no
|
||
// earlier warning and only costs statfs calls. 03:30 is deliberately BEFORE the nightly app-data
|
||
// legs (db-dump / tier2 / offbox), so a customer who is about to lose a backup to lack of space
|
||
// hears about it while there is still a night's margin, rather than after the failure.
|
||
// The interval is NOT a cooldown: repeats are impossible because the check is edge-triggered per
|
||
// filesystem, and the hub owns cooldown regardless.
|
||
fillWatcher := fillwatch.New(
|
||
filepath.Join(cfg.Paths.DataDir, "fillwatch-state.json"), logger,
|
||
func() []fillwatch.Target { return fillTargets(cfg, sett) },
|
||
func(p string) *fillwatch.Usage {
|
||
di := system.GetDiskUsage(p)
|
||
if di == nil {
|
||
return nil // §8.4 — unreadable is NOT full; never fabricate a zero here
|
||
}
|
||
return &fillwatch.Usage{
|
||
UsedPercent: di.UsedPercent, AvailGB: di.AvailGB,
|
||
UsedGB: di.UsedGB, TotalGB: di.TotalGB,
|
||
}
|
||
})
|
||
if notifier != nil {
|
||
fillWatcher.SetNotify(func(e fillwatch.Event) {
|
||
// The CUSTOMER's event — deliberately not operator-only. A customer can free space,
|
||
// delete files or add a drive, so this alert is theirs (D-c). The dynamic Hungarian
|
||
// message carries the label and the free space; the hub has NO customerMessages entry for
|
||
// these two types precisely so the template cannot discard them.
|
||
notifier.PushEvent(e.Band.EventType(), e.Band.Severity(), e.Message, map[string]any{
|
||
"path": e.Target.Path, "label": e.Target.Label,
|
||
"used_percent": e.Usage.UsedPercent, "avail_gb": e.Usage.AvailGB,
|
||
"total_gb": e.Usage.TotalGB, "band": e.Band.String(),
|
||
})
|
||
})
|
||
}
|
||
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() })
|
||
|
||
// AND ONCE SHORTLY AFTER STARTUP. A box that BOOTS with a filesystem already over the line must
|
||
// warn now, not up to 24 hours later — that is the R-100 shape, a real fault visible only after a
|
||
// deadline elapses. The hub's own checkers handle exactly this case deliberately ("already-breached
|
||
// keys are left UNSEEDED at init so their first Check emits" — the F2 lesson, storage_fill.go), and
|
||
// a daily-only schedule here would have been the same gap one component over.
|
||
//
|
||
// It is SAFE because the check is edge-triggered against PERSISTED state: a filesystem the customer
|
||
// has already been warned about stays silent, so this adds a warning only where one is genuinely
|
||
// owed. The delay lets mounts settle and the drive gate tick first, so a drive that is merely slow
|
||
// to come back reads as unreadable (§8.4, skipped) rather than as anything at all.
|
||
go func() {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(fillWatchStartupDelay):
|
||
}
|
||
if err := fillWatcher.Check(); err != nil {
|
||
logger.Printf("[WARN] [fillwatch] startup check failed: %v", err)
|
||
}
|
||
}()
|
||
|
||
// --- Central hub reporting schedule ---
|
||
if hubPusher != nil {
|
||
if cfg.Hub.Enabled {
|
||
pushInterval, err := time.ParseDuration(cfg.Hub.PushInterval)
|
||
if err != nil {
|
||
pushInterval = 15 * time.Minute
|
||
}
|
||
sched.Every("hub-report", pushInterval, func(ctx context.Context) error {
|
||
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
|
||
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
|
||
if err := hubPusher.Push(r); err != nil {
|
||
return err
|
||
}
|
||
// Drain pending events (e.g., DR recovery completed) after successful push
|
||
if events := sett.DrainPendingEvents(); len(events) > 0 {
|
||
for _, ev := range events {
|
||
notifier.Notify(ev.EventType, ev.Severity, ev.Message, ev.Details)
|
||
}
|
||
logger.Printf("[INFO] Drained %d pending events to Hub", len(events))
|
||
}
|
||
return nil
|
||
})
|
||
logger.Printf("[INFO] Hub reporting enabled (every %s to %s)", pushInterval, cfg.Hub.URL)
|
||
} else {
|
||
logger.Printf("[INFO] Hub reporting disabled — will send disabled notification to %s", cfg.Hub.URL)
|
||
}
|
||
}
|
||
|
||
// Self-update scheduler jobs
|
||
if cfg.SelfUpdate.Enabled && updater != nil {
|
||
// Periodic version check (populates UI, never triggers update)
|
||
checkInterval, ciErr := time.ParseDuration(cfg.SelfUpdate.CheckInterval)
|
||
if ciErr != nil {
|
||
checkInterval = 6 * time.Hour
|
||
}
|
||
sched.Every("selfupdate-check", checkInterval, func(ctx context.Context) error {
|
||
result := updater.CheckForUpdate()
|
||
if result.UpdateAvailable {
|
||
logger.Printf("[INFO] Update available: %s -> %s", result.CurrentVersion, result.LatestVersion)
|
||
}
|
||
return nil
|
||
})
|
||
|
||
// Auto-update (daily, fires after typical backup completion)
|
||
if cfg.SelfUpdate.AutoUpdate {
|
||
sched.Daily("selfupdate-auto", cfg.SelfUpdate.AutoUpdateTime, func(ctx context.Context) error {
|
||
result := updater.CheckForUpdate()
|
||
if !result.UpdateAvailable {
|
||
return nil
|
||
}
|
||
if err := updater.TriggerUpdate("auto"); err != nil {
|
||
logger.Printf("[WARN] Auto-update skipped: %v", err)
|
||
}
|
||
return nil
|
||
})
|
||
}
|
||
}
|
||
|
||
// Storage watchdog (disk disconnect/reconnect detection) has moved to the host
|
||
// agent (slice 8C) — the controller no longer owns disk-level monitoring.
|
||
|
||
// --- Asset syncer (download from Hub) ---
|
||
var assetsSyncer *assets.Syncer
|
||
if cfg.Hub.Enabled && cfg.Assets.SyncEnabled && cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
|
||
assetsDir := filepath.Join(cfg.Paths.DataDir, "assets")
|
||
assetsSyncer = assets.New(cfg.Hub.URL, cfg.Hub.APIKey, assetsDir, "/usr/share/felhom/assets", logger, cfg.Logging.Level == "debug")
|
||
go func() {
|
||
time.Sleep(10 * time.Second)
|
||
if err := assetsSyncer.Sync(ctx); err != nil {
|
||
logger.Printf("[WARN] Initial asset sync failed: %v", err)
|
||
}
|
||
}()
|
||
sched.Daily("asset-sync", cfg.Assets.SyncSchedule, func(ctx context.Context) error {
|
||
return assetsSyncer.Sync(ctx)
|
||
})
|
||
logger.Printf("[INFO] Asset sync enabled (daily at %s from Hub)", cfg.Assets.SyncSchedule)
|
||
}
|
||
|
||
// --- Startup self-test ---
|
||
selfTestResult := selftest.Run(cfg, sett, logger)
|
||
|
||
sched.Start(ctx)
|
||
defer sched.Stop()
|
||
|
||
// Generate recovery info file if retrieval password is set
|
||
if rp := sett.GetRetrievalPassword(); rp != "" {
|
||
go func() {
|
||
info := recovery.Info{
|
||
CustomerID: cfg.Customer.ID,
|
||
RetrievalPassword: rp,
|
||
HubURL: cfg.Hub.URL,
|
||
SupportEmail: "support@felhom.eu",
|
||
SupportURL: "https://felhom.eu/kapcsolat",
|
||
}
|
||
if err := recovery.GenerateRecoveryFile(info, Version, cfg.Paths.DataDir); err != nil {
|
||
logger.Printf("[WARN] Failed to generate recovery-info.txt: %v", err)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// Fire startup pings + hub report immediately (don't wait for first scheduler tick)
|
||
go func() {
|
||
time.Sleep(5 * time.Second) // Let all subsystems fully initialize
|
||
|
||
// Push controller startup event to Hub
|
||
notifier.NotifyControllerStarted(Version, map[string]interface{}{
|
||
"selftest_pass": selfTestResult.Pass,
|
||
"selftest_warn": selfTestResult.Warn,
|
||
"selftest_fail": selfTestResult.Fail,
|
||
})
|
||
|
||
// Hub report
|
||
if hubPusher != nil {
|
||
if cfg.Hub.Enabled {
|
||
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
|
||
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
|
||
var pushErr error
|
||
for attempt := 1; attempt <= 3; attempt++ {
|
||
pushErr = hubPusher.Push(r)
|
||
if pushErr == nil {
|
||
logger.Println("[INFO] Startup hub report sent")
|
||
break
|
||
}
|
||
logger.Printf("[WARN] Startup hub report attempt %d/3 failed: %v", attempt, pushErr)
|
||
if attempt < 3 {
|
||
time.Sleep(15 * time.Second)
|
||
}
|
||
}
|
||
if pushErr != nil {
|
||
logger.Printf("[WARN] Startup hub report failed after 3 attempts — next scheduled push in %s", cfg.Hub.PushInterval)
|
||
}
|
||
} else {
|
||
// Send a minimal "disabled" notification so hub knows reporting is intentionally off
|
||
r := &report.Report{
|
||
Version: 1,
|
||
CustomerID: cfg.Customer.ID,
|
||
CustomerName: cfg.Customer.Name,
|
||
ControllerVersion: Version,
|
||
Timestamp: time.Now().UTC(),
|
||
ReportingDisabled: true,
|
||
Health: report.HealthReport{Status: "disabled", Issues: []string{}, Warnings: []string{}},
|
||
Stacks: report.StacksReport{Deployed: []string{}, Available: []string{}},
|
||
Containers: report.ContainerReport{List: []report.ContainerDetailReport{}},
|
||
}
|
||
hubPusher.PushOnce(r)
|
||
}
|
||
}
|
||
|
||
// Initial self-update check (so settings page shows version info quickly)
|
||
if updater != nil {
|
||
time.Sleep(25 * time.Second) // Additional delay after hub report
|
||
result := updater.CheckForUpdate()
|
||
if result.UpdateAvailable {
|
||
logger.Printf("[INFO] Startup: update available %s -> %s", result.CurrentVersion, result.LatestVersion)
|
||
} else if result.Error != "" {
|
||
logger.Printf("[DEBUG] Startup version check: %s", result.Error)
|
||
}
|
||
}
|
||
}()
|
||
|
||
// Initial backup cache population (don't block startup)
|
||
if cfg.Backup.Enabled && backupMgr != nil {
|
||
go func() {
|
||
curDB, _, _ := backupwindow.LegTimes(backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule))
|
||
backupMgr.RefreshCache(scheduler.NextDailyRun(curDB))
|
||
}()
|
||
}
|
||
|
||
// Sync notification preferences to hub on startup (handles hub DB rebuild recovery)
|
||
if notifier.IsEnabled() {
|
||
go func() {
|
||
prefs := sett.GetNotificationPrefs()
|
||
if prefs.Email != "" {
|
||
if err := notifier.SyncPreferences(prefs.Email, prefs.EnabledEvents, prefs.CooldownHours); err != nil {
|
||
logger.Printf("[WARN] Failed to sync notification preferences on startup: %v", err)
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
|
||
// Initial alert refresh (so alerts appear immediately, not after first 5min health check)
|
||
go func() {
|
||
report := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), sett.GetSMBSettings(), logger)
|
||
alertMgr.Refresh(report, cfg, backupMgr, false, "")
|
||
}()
|
||
|
||
// --- Out-of-cycle report trigger (v0.139.0, Direction 1) ---
|
||
// ONE canonical fire closure (full BuildReport + Claimed + Push) behind a debounced,
|
||
// coalescing trigger — user actions with hub-side effects (geo, escrow claim, settings
|
||
// save, offsite toggle, app deploy/remove, customer claim) round-trip in seconds instead
|
||
// of the next ~15-min cycle. The scheduled hub-report job above stays the reconciliation
|
||
// backbone; the trigger is best-effort on top (its failures degrade to the cycle).
|
||
// nil when hub reporting is off → the api/web seams stay unset (strict no-op).
|
||
var reportTrigger *report.Trigger
|
||
if hubPusher != nil && cfg.Hub.Enabled {
|
||
fireReport := func() error {
|
||
rep := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
|
||
rep.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
|
||
return hubPusher.Push(rep)
|
||
}
|
||
reportTrigger = report.NewTrigger(fireReport, logger)
|
||
go reportTrigger.Run(ctx)
|
||
|
||
// Direction-2 immediate-sync (v0.140.0): hold a hanging GET against the hub's wait channel
|
||
// and fire the trigger the instant operator intent moves — so an operator save round-trips in
|
||
// seconds instead of on the next ~15-min cycle. Reuses the SAME hub URL + key as the pusher;
|
||
// no new config keys. Gated on the trigger existing (hub reporting enabled). The ACK from the
|
||
// fired report delivers everything through the unchanged machinery.
|
||
waiter := report.NewWaiter(cfg.Hub.URL, cfg.Hub.APIKey, reportTrigger.Fire, logger)
|
||
go waiter.Run(ctx)
|
||
}
|
||
|
||
// --- Initialize API router ---
|
||
apiRouter := api.NewRouter(cfg, *configPath, sett, stackMgr, syncer, cpuCollector, backupMgr, metricsStore, updater, notifier, logger)
|
||
if reportTrigger != nil {
|
||
// Out-of-cycle, non-blocking hub report push (geo changes + app deploy/remove) so
|
||
// the hub reflects the new state immediately instead of after the next ~15-min cycle.
|
||
apiRouter.SetReportPushTrigger(reportTrigger.Fire)
|
||
}
|
||
if assetsSyncer != nil {
|
||
apiRouter.SetAssetsSyncer(assetsSyncer)
|
||
}
|
||
|
||
// --- Initialize Cloudflare geo-restriction ---
|
||
var geoSync *cf.GeoSyncManager
|
||
if cfg.Infrastructure.CFAPIToken != "" {
|
||
cfClient := cf.New(cfg.Infrastructure.CFAPIToken, logger, cfg.Logging.Level == "debug")
|
||
geoStacks := &geoStackAdapter{mgr: stackMgr, domain: cfg.Customer.Domain}
|
||
geoSync = cf.NewGeoSyncManager(cfClient, sett, cfg.Customer.Domain, geoStacks, logger)
|
||
geoSync.SetDebug(cfg.Logging.Level == "debug")
|
||
apiRouter.SetGeoSync(geoSync)
|
||
|
||
// Re-sync geo rules when apps are deployed/removed
|
||
apiRouter.OnGeoRelevantChange = func() {
|
||
geo := sett.GetGeoRestriction()
|
||
if geo != nil && geo.Enabled {
|
||
if err := geoSync.Sync(context.Background()); err != nil {
|
||
logger.Printf("[WARN] Geo sync after app change failed: %v", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Periodic verification every 6 hours
|
||
sched.Every("geo-verify", 6*time.Hour, func(ctx context.Context) error {
|
||
geo := sett.GetGeoRestriction()
|
||
if geo == nil || !geo.Enabled {
|
||
return nil
|
||
}
|
||
return geoSync.Sync(ctx)
|
||
})
|
||
|
||
// Initial sync (delayed, non-blocking)
|
||
go func() {
|
||
time.Sleep(15 * time.Second)
|
||
if geo := sett.GetGeoRestriction(); geo != nil && geo.Enabled {
|
||
if err := geoSync.Sync(context.Background()); err != nil {
|
||
logger.Printf("[WARN] Initial geo sync failed: %v", err)
|
||
}
|
||
}
|
||
}()
|
||
|
||
logger.Printf("[INFO] Geo-restriction support enabled (CF API token configured)")
|
||
}
|
||
|
||
// --- Initialize integration manager ---
|
||
integrationStacks := &integrationStackAdapter{mgr: stackMgr}
|
||
integrationMgr := integrations.NewManager(sett, integrationStacks, cfg.Customer.Domain, cfg.Paths.StacksDir, encKey, logger)
|
||
integrationMgr.SetDebug(cfg.Logging.Level == "debug")
|
||
apiRouter.SetIntegrationManager(integrationMgr)
|
||
|
||
// --- Initialize app exporter ---
|
||
exportProv := &exportAdapter{mgr: stackMgr, encKey: encKey}
|
||
appExporter := appexport.NewExporter(exportProv, logger, Version)
|
||
appExporter.SetDebug(cfg.Logging.Level == "debug")
|
||
// R-166: the exporter stops apps too (export with "stop the app first"), so it shares the backup
|
||
// manager's ONE marker file rather than opening a second one — one file, one recovery. Without
|
||
// this the export path would be the uncovered sibling of two covered ones, which is how a reader
|
||
// concludes the whole class is handled (§2.2).
|
||
appExporter.SetStopGuard(exportStopGuard{g: appStopGuard})
|
||
apiRouter.SetDebug(cfg.Logging.Level == "debug")
|
||
|
||
// --- Initialize web server ---
|
||
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, sched, sett, alertMgr, notifier, updater, logger, Version)
|
||
// Migration done-hook: a decommission-initiated migration finalizes the source decommission on
|
||
// success (soft-mark + agent). Wire it before RecoverMigration so a resumed one still finalizes.
|
||
stackMgr.SetMigrationDoneHook(webServer.OnMigrationDone)
|
||
stackMgr.RecoverMigration(ctx)
|
||
webServer.SetEncryptionKey(encKey)
|
||
webServer.SetAppExporter(appExporter)
|
||
// Disk-health degradation check (v0.169.0): every 6h, compare each physical disk's SMART verdict
|
||
// against the in-memory baseline and emit disk_health_degraded on a degradation only (first run
|
||
// baselines silently; recovery/UNKNOWN never notify). Only on a provisioned guest (an agent to
|
||
// read /disks from); the check no-ops gracefully if the agent is unreachable.
|
||
if cfg.LocalAPI.Endpoint != "" {
|
||
sched.Every("disk-health-check", 6*time.Hour, webServer.RunDiskHealthCheck)
|
||
}
|
||
// Browser .fab upload (v0.128.0): upload state is in-memory, so a restart strands the .part —
|
||
// GC stray part files in every registered drive's exports dir at startup.
|
||
webServer.CleanupStaleUploadParts()
|
||
// Escrow wizard (v0.127.0): the Scenario-F stale-blob flag feeds the Távoli mentés card.
|
||
if escrowConfirmer != nil {
|
||
webServer.SetEscrowStale(escrowConfirmer.StaleBlob)
|
||
// R-193: the recovery screen may state WHEN the hub sealed the package, and nothing more,
|
||
// before a code is entered. A timestamp, never a secret.
|
||
webServer.SetEscrowSealedAt(escrowConfirmer.SealedAt)
|
||
}
|
||
// R-219 / R-218: the unlock brings the off-site tier up between placing the recovered key and
|
||
// reading the repository. Without it the listing the screen promises can never render on the
|
||
// pristine rebuilt box — no key ⇒ no target ⇒ no inventory — and the apply-bridge's only other
|
||
// trigger is a config refresh or a restart, neither of which the customer can cause.
|
||
if offsiteBridge != nil {
|
||
webServer.SetRecoveryTierUp(offsiteBridge.Reconcile)
|
||
}
|
||
webServer.SetIntegrationManager(integrationMgr)
|
||
if reportTrigger != nil {
|
||
// Out-of-cycle report push after hub-relevant user actions (escrow claim, settings
|
||
// save, offsite config/toggle, customer claim) — same debounced trigger as the api
|
||
// router's; nil (hub reporting off) leaves the seam a strict no-op.
|
||
webServer.SetReportTrigger(reportTrigger.Fire)
|
||
}
|
||
if quiesceLoop != nil {
|
||
webServer.SetBackupTrigger(quiesceLoop) // "Mentés most" → app-consistent backup via the quiesce loop
|
||
}
|
||
if mailShim != nil {
|
||
webServer.SetMailShim(mailShim) // settings toggle starts/stops the app-email shim at runtime
|
||
}
|
||
if assetsSyncer != nil {
|
||
webServer.SetAssetsSyncer(assetsSyncer)
|
||
}
|
||
if hubPusher != nil {
|
||
webServer.SetHubPushStatus(func() web.HubPushStatusData {
|
||
s := hubPusher.GetStatus()
|
||
return web.HubPushStatusData{
|
||
LastAttempt: s.LastAttempt,
|
||
LastSuccess: s.LastSuccess,
|
||
LastError: s.LastError,
|
||
Consecutive: s.Consecutive,
|
||
}
|
||
})
|
||
}
|
||
if logBuffer != nil {
|
||
webServer.SetLogBuffer(logBuffer)
|
||
}
|
||
webServer.SetStartTime(startTime)
|
||
|
||
// Controller→agent channel health (self-health slice). A ~60s probe of the local-API channel via
|
||
// the PRODUCTION memoized client (webServer.ProbeAgentChannel), classified + debounced, alerting the
|
||
// operator + dashboard on a transition. Only on a provisioned guest (endpoint set) — mirrors
|
||
// probeLocalAPI's guard. Registered after webServer (it owns the memoized client); sched.Every
|
||
// launches the job immediately since the scheduler is already started.
|
||
if cfg.LocalAPI.Endpoint != "" {
|
||
chSink := channelSink{notifier: notifier, alertMgr: alertMgr}
|
||
chChecker := channelhealth.New(webServer.ProbeAgentChannel, chSink, logger)
|
||
sched.Every("agent-channel-health", 60*time.Second, chChecker.Check)
|
||
}
|
||
|
||
// local_api endpoint drift (R-77, from the 2026-07-25 outage): controller.yaml and bootstrap.json
|
||
// can disagree indefinitely and silently — the island migration rewrote the latter and the
|
||
// controller kept dialling the former for 17.5 h, alerting only "agent unreachable". This NAMES
|
||
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins).
|
||
//
|
||
// Startup-only is sufficient and correct: both files are read at boot and neither changes under a
|
||
// running controller, so a periodic re-check would add noise without adding signal.
|
||
if d := bootstrap.DetectEndpointDrift(*configPath, cfg, logger); d != nil {
|
||
alertMgr.SetEndpointDriftAlert(true, d.HungarianMessage())
|
||
if notifier != nil {
|
||
notifier.NotifyEndpointDrift(d.EnglishMessage(), d.FingerprintAgrees)
|
||
}
|
||
}
|
||
|
||
// Wire debug callbacks (only in debug mode)
|
||
if cfg.Logging.Level == "debug" {
|
||
dc := &web.DebugCallbacks{}
|
||
if hubPusher != nil {
|
||
dc.TriggerHubReportPush = func() error {
|
||
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), sett.GetSMBSettings(), logger)
|
||
r.Claimed = sett.GetClaimed() // v0.122.0 (F-4): set-only claim flag for the hub
|
||
return hubPusher.Push(r)
|
||
}
|
||
}
|
||
dc.HubConnectivityTest = func() (int, int64, error) {
|
||
start := time.Now()
|
||
client := &http.Client{Timeout: 10 * time.Second}
|
||
resp, err := client.Get(cfg.Hub.URL + "/healthz")
|
||
latency := time.Since(start).Milliseconds()
|
||
if err != nil {
|
||
return 0, latency, err
|
||
}
|
||
resp.Body.Close()
|
||
return resp.StatusCode, latency, nil
|
||
}
|
||
if cfg.Git.RepoURL != "" {
|
||
dc.GiteaConnectivityTest = func() (int, int64, error) {
|
||
start := time.Now()
|
||
client := &http.Client{Timeout: 5 * time.Second}
|
||
resp, err := client.Head(cfg.Git.RepoURL)
|
||
latency := time.Since(start).Milliseconds()
|
||
if err != nil {
|
||
return 0, latency, err
|
||
}
|
||
resp.Body.Close()
|
||
return resp.StatusCode, latency, nil
|
||
}
|
||
}
|
||
dc.GetTelemetryPreview = func() ([]report.AppTelemetry, error) {
|
||
return report.BuildAppTelemetryForDebug(stackMgr, metricsStore, logger), nil
|
||
}
|
||
webServer.SetDebugCallbacks(dc)
|
||
}
|
||
|
||
// Drive migration (full-drive move) has moved to the host agent (slice 8C);
|
||
// the controller no longer runs a DriveMigrator.
|
||
|
||
// --- Build HTTP mux ---
|
||
mux := http.NewServeMux()
|
||
|
||
// API routes (no auth for health endpoint, auth for everything else)
|
||
mux.HandleFunc("/api/health", apiRouter.HealthHandler)
|
||
// Disk management API — thin proxy to the host agent (slice 8C). The agent owns
|
||
// disk execution; the controller forwards list/assign/eject/format.
|
||
mux.Handle("/api/disks", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
|
||
mux.Handle("/api/disks/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
|
||
// Guided storage provisioning (init/attach/eject orchestration over the agent disk API + registry).
|
||
mux.Handle("/api/storage/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeStorageAPI))))
|
||
// LAN network-sharing folder picker (R-7 slice 1). Must live here, not in the web ServeHTTP
|
||
// switch: the /api/ subtree is routed on this mux, so a case there is shadowed and 401s.
|
||
mux.Handle("/api/sharing/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeSharingAPI))))
|
||
// Guest RAM resize (v0.143.0, R-24): read current allocation/bounds + apply a bounded resize.
|
||
mux.Handle("/api/system/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeSystemAPI))))
|
||
// Standalone full-server (guest) restart — the "Kiszolgáló újraindítása" maintenance affordance,
|
||
// a sibling to the controller-only /api/selfrestart. Reuses the agent GuestReboot primitive.
|
||
mux.Handle("/api/server/reboot", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.HandleServerReboot))))
|
||
// Whole-guest (appliance) backup visibility + manual trigger. Distinct prefix from apiRouter's
|
||
// app-data /api/backup/{run,status} (DB dumps) to avoid shadowing the /api/ catch-all subtree.
|
||
mux.Handle("/api/guest-backup/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeBackupAPI))))
|
||
// Host metrics API — thin proxy to the host agent (slice 9). Read-only host-wide health +
|
||
// per-storage capacity for the monitoring view; the de-privileged controller can't read the
|
||
// host itself. GET only, so no CSRF wrapper needed.
|
||
mux.Handle("/api/host-metrics", webServer.RequireAuth(http.HandlerFunc(webServer.ServeHostMetricsAPI)))
|
||
// App export/import API routes handled by web server
|
||
mux.Handle("/api/export/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeExportAPI))))
|
||
// Debug API routes handled by web server (debug-mode gating inside handler)
|
||
mux.Handle("/api/debug/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDebugAPI))))
|
||
// Escrow ceremony wizard API (v0.127.0) — session auth + CSRF on POSTs; the claim response is
|
||
// the ONLY surface the recovery code R ever crosses (no-store, never logged).
|
||
mux.Handle("/api/escrow/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeEscrowAPI))))
|
||
// Self-update API — accepts session auth OR hub API key (for external triggering)
|
||
// CsrfProtect exempts Bearer-token requests automatically.
|
||
mux.Handle("/api/selfupdate/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
|
||
// Config API — accepts session auth OR hub API key (for Hub config push)
|
||
mux.Handle("/api/config/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
|
||
// Geo API — accepts session auth OR hub API key (for Hub geo-disable)
|
||
mux.Handle("/api/geo/", selfUpdateAuthMiddleware(cfg, webServer, webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
|
||
mux.Handle("/api/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(apiRouter.ServeHTTP))))
|
||
|
||
// Web UI routes (auth required)
|
||
mux.Handle("/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeHTTP))))
|
||
|
||
// --- Start HTTP server ---
|
||
server := &http.Server{
|
||
Addr: cfg.Web.Listen,
|
||
Handler: webServer.CatchAllMiddleware(mux),
|
||
ReadTimeout: 30 * time.Second,
|
||
WriteTimeout: 60 * time.Second,
|
||
IdleTimeout: 120 * time.Second,
|
||
}
|
||
|
||
sigCh := make(chan os.Signal, 1)
|
||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||
|
||
go func() {
|
||
sig := <-sigCh
|
||
logger.Printf("[INFO] Received signal %v, shutting down...", sig)
|
||
// fix-6: spill the debug ring on a clean shutdown so a `systemctl restart` / container recreate
|
||
// preserves the pre-restart window (the periodic spill already covers a hard crash to ≤60 s).
|
||
if err := logBuffer.SpillTo(ringSpillPath); err != nil {
|
||
logger.Printf("[WARN] debug-ring spill on shutdown failed: %v", err)
|
||
}
|
||
cancel()
|
||
if mailShim != nil {
|
||
mailShim.Close()
|
||
}
|
||
|
||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer shutdownCancel()
|
||
|
||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||
logger.Printf("[ERROR] HTTP server shutdown error: %v", err)
|
||
}
|
||
}()
|
||
|
||
logger.Printf("[INFO] Web UI listening on %s", cfg.Web.Listen)
|
||
if err := server.ListenAndServe(); err != http.ErrServerClosed {
|
||
logger.Fatalf("[FATAL] HTTP server error: %v", err)
|
||
}
|
||
|
||
logger.Println("[INFO] felhom-controller stopped")
|
||
}
|
||
|
||
// selfUpdateAuthMiddleware allows access via session auth (normal UI) OR hub API key bearer token (external).
|
||
// deadAppBootGrace is the startup settle window before fix-3 evaluates deployed-app run states — apps
|
||
// legitimately take 30–60 s to come up, so a shorter grace would false-alarm during the controller's
|
||
// own boot. After the grace, an app that still isn't running alerts (the F11 dead-at-boot case).
|
||
const deadAppBootGrace = 90 * time.Second
|
||
|
||
// deadAppHeartbeatEvery is how many 30 s scans pass between deadapp heartbeat lines (F-OBS).
|
||
// 20 scans = one line per ~10 minutes: frequent enough that a stalled detector is obvious well
|
||
// inside the 180 s alarm grace it feeds, and 144 lines/day instead of 2880.
|
||
const deadAppHeartbeatEvery = 20
|
||
|
||
// deadAppScans counts completed deadapp scans since boot. Single-goroutine (the scheduler runs
|
||
// each job serially), so it needs no lock.
|
||
var deadAppScans int
|
||
|
||
// noteDeadAppScan emits the F-OBS heartbeat every deadAppHeartbeatEvery-th scan, at [INFO] so it
|
||
// survives a default `logging.level: info` box. Extracted from the job closure so a test can assert
|
||
// the OBSERVABLE (the line, at info) rather than merely that the scan function ran — which is the
|
||
// distinction F-OBS is about.
|
||
func noteDeadAppScan(logger *log.Logger, scans, evaluated, down int) {
|
||
if logger == nil || deadAppHeartbeatEvery <= 0 || scans%deadAppHeartbeatEvery != 0 {
|
||
return
|
||
}
|
||
logger.Printf("[INFO] [deadapp] check alive: %d scans since boot, %d deployed app(s) evaluated, %d currently down",
|
||
scans, evaluated, down)
|
||
}
|
||
|
||
// bootReconcileSettle is the delay before the FIRST observation. It lets the initial scan, the first
|
||
// status refresh and the two crash recoveries land before the R-52 sweep decides what "down" means.
|
||
var bootReconcileSettle = 5 * time.Second
|
||
|
||
// ── R-157 mechanism A: the window, and why these three numbers ───────────────────────────────────
|
||
//
|
||
// Until v0.190.0 the sweep looked exactly ONCE, at T+5 s, and returned. At T+5 s docker is still
|
||
// restoring containers after a hard reset, so an app that has not yet settled into a down state is
|
||
// not a candidate — and because the sweep never looked again, it stayed down. Measured failing on
|
||
// THREE OF SIX hard resets (CAMPAIGN-10). The predicate was never the problem; the single
|
||
// observation was.
|
||
//
|
||
// The fix is a window that SETTLES rather than a timer that runs forever (§5: an unbounded loop
|
||
// papers over a genuinely broken app and hammers docker). Three constants, each chosen against the
|
||
// 90 s dead-app boot grace:
|
||
//
|
||
// - bootReconcileSample = 5 s. Fine enough that a container settling at T+40 s is seen within one
|
||
// sample, coarse enough that a quiet boot costs ~12 cheap GetStacks() calls, not hundreds.
|
||
// - bootReconcileStableFor = 3 consecutive identical samples (15 s of no change) before the fleet
|
||
// is called settled. One sample cannot distinguish "settled" from "sampled between two docker
|
||
// events"; three spans the gap between a container exiting and its restart policy re-creating it.
|
||
// - bootReconcileBudget = 50 s. THE BINDING CONSTRAINT, and it is arithmetic, not taste:
|
||
// bootReconcileSettle (5 s) + budget (50 s) + ONE DefaultRetryDelay (30 s) inside the final
|
||
// sweep = 85 s, which must stay under deadAppBootGrace (90 s) so a recovery that works is
|
||
// SILENT. 60 s was the first choice and TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace
|
||
// rejected it at 95 s — the test is the reason this number is 50 and not a round 60.
|
||
// Extending the grace to make a bigger budget fit was rejected (§8.3): that hides a late
|
||
// recovery rather than reporting it. A window that genuinely overruns is reported instead —
|
||
// see recordLateRecovery below.
|
||
//
|
||
// The window ends on WHICHEVER COMES FIRST — settled, or budget exhausted — and the log says which,
|
||
// because "settled and found nothing" and "ran out of time still churning" are different facts about
|
||
// the box and must not read the same (the v0.91.2 lesson).
|
||
var (
|
||
bootReconcileSample = 5 * time.Second
|
||
bootReconcileStableFor = 3
|
||
bootReconcileBudget = 50 * time.Second
|
||
)
|
||
|
||
// bootReconcileFn is the R-52 sweep, a package var purely so the wiring below is testable from
|
||
// package main (the v0.154.0 / v0.91.0 lesson: a seam proven only through injection proves the
|
||
// component and not the caller).
|
||
var bootReconcileFn = func(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) bootrecon.Result {
|
||
r := bootrecon.New(mgr, logger)
|
||
// R-171: the sweep must not start an app whose data drive is absent. Wired HERE, at the one
|
||
// place the sweep is constructed, so there is no path that builds an ungated reconciler.
|
||
if sm, ok := mgr.(*stacks.Manager); ok {
|
||
r.SetDriveGate(bootDriveGate{drive: driveStartGate{mgr: sm, sett: bootDriveSettings}})
|
||
}
|
||
return r.Run(ctx)
|
||
}
|
||
|
||
// bootDriveSettings / bootStartHolders are what the boot start gate reads. Init-only, set in main()
|
||
// before the reconcile goroutine is launched; both are nil-safe (see MayStart).
|
||
var (
|
||
bootDriveSettings *settings.Settings
|
||
bootQuiesceLoop *quiesce.Loop
|
||
bootAppStopGuard *backup.AppStopGuard
|
||
)
|
||
|
||
// bootDriveGate answers bootrecon.StartGate for the real controller. It enforces §8.2: an app that
|
||
// something else is deliberately holding must NOT be started by the boot sweep.
|
||
//
|
||
// THE THREE HOLDERS, in the order they are checked. The first two only became reachable when R-157
|
||
// mechanism A widened the window — the old T+5 s single sweep never overlapped a quiesce or a
|
||
// running app-data operation, and that is exactly why widening it needed these:
|
||
//
|
||
// 1. QUIESCE — the whole-guest backup loop stops app stacks and restarts exactly the ones it
|
||
// stopped. Starting one mid-backup would put a running app inside a snapshot that is supposed to
|
||
// be clean-shutdown-consistent, which is the entire point of quiescing. `SuppressedStacks` is
|
||
// the set it already publishes for precisely this "an app WE stopped is not a fault" question,
|
||
// so reusing it means the two cannot drift.
|
||
// 2. THE APP-STOP GUARD — a volume dump / offbox reconstitute / .fab export that is CURRENTLY
|
||
// holding an app down. Its own Recover already ran to completion before this goroutine started,
|
||
// so the marker seen here belongs to an operation running NOW, not to a crashed one.
|
||
// 3. THE DRIVE — R-171. Two questions, because they fail in opposite directions and neither alone
|
||
// is sufficient at boot:
|
||
// • the settings flags (`Disconnected`/`Decommissioned`) — the SAME signal the API's own
|
||
// `startGatedByMissingDrive` uses, so the customer's path and the sweep cannot disagree about
|
||
// whether an app may start. But they are the drive gate's bookkeeping, and inside the boot
|
||
// window that gate may not have ticked yet, so a genuinely absent drive can still read as
|
||
// connected.
|
||
// • `Manager.DriveLive` — the live mountpoint check, the SAME `isMountPoint` seam the userdata
|
||
// belt already uses. It answers immediately and needs no gate tick.
|
||
//
|
||
// Fail-safe per bootrecon.StartGate's contract: anything that cannot be determined returns false.
|
||
type bootDriveGate struct {
|
||
drive driveStartGate
|
||
}
|
||
|
||
func (g bootDriveGate) MayStart(stackName string) (bool, string) {
|
||
// 1. quiesce (nil-safe on an unprovisioned guest: SuppressedStacks returns nil)
|
||
if bootQuiesceLoop.SuppressedStacks()[stackName] {
|
||
return false, "a whole-guest backup (quiesce) is holding it — the quiesce loop restarts its own stacks"
|
||
}
|
||
// 2. an app-data operation in flight
|
||
for _, held := range bootAppStopGuard.HeldStacks() {
|
||
if held == stackName {
|
||
return false, "an app-data operation is holding it — the app-stop guard restarts it when the operation ends"
|
||
}
|
||
}
|
||
// 3. the drive — the SHARED predicate, so this gate and the app-stop guard's crash recovery
|
||
// (R-174) cannot disagree about whether an app's drive is available.
|
||
return g.drive.MayStart(stackName)
|
||
}
|
||
|
||
// driveStartGate is holder #3 of bootDriveGate, EXTRACTED so it has two callers and one
|
||
// implementation (R-174).
|
||
//
|
||
// WHY IT IS SEPARATE FROM bootDriveGate RATHER THAN REUSED WHOLE. The app-stop guard's Recover needs
|
||
// exactly this question and NOT the other two holders:
|
||
//
|
||
// - Holder #2 reads `bootAppStopGuard.HeldStacks()`, which during Recover is the guard's OWN
|
||
// marker — the very stacks being recovered. Reusing bootDriveGate there would refuse every
|
||
// recovery it was meant to perform, self-referentially.
|
||
// - Holders #1 and #2 read package-level vars assigned in main() at the `bootDriveSettings` block,
|
||
// which runs AFTER `appStopGuard.Recover()`. They are nil at recovery time, so a whole-gate
|
||
// reuse would be correct only by accident of nil-safety — and would silently invert the moment
|
||
// anyone moved either line. This project has shipped that class of accident before.
|
||
//
|
||
// Fail-safe per bootrecon.StartGate's contract: anything that cannot be determined returns false.
|
||
type driveStartGate struct {
|
||
mgr *stacks.Manager
|
||
sett *settings.Settings
|
||
}
|
||
|
||
func (g driveStartGate) MayStart(stackName string) (bool, string) {
|
||
if g.mgr == nil {
|
||
return false, "no stack manager wired — the drive cannot be determined"
|
||
}
|
||
cfg := g.mgr.LoadAppConfigByName(stackName)
|
||
if cfg == nil {
|
||
// CANNOT DETERMINE. A deployed app whose app.yaml will not load cannot have its drive
|
||
// resolved, so the fail-safe direction applies rather than a hopeful start.
|
||
return false, "app.yaml could not be read"
|
||
}
|
||
hdd := cfg.Env["HDD_PATH"]
|
||
if hdd == "" {
|
||
return true, "" // SSD-resident: there is no external drive to be absent (mirrors the API gate)
|
||
}
|
||
if g.sett != nil {
|
||
for _, sp := range g.sett.GetStoragePaths() {
|
||
if sp.Path != hdd {
|
||
continue
|
||
}
|
||
if sp.Decommissioned {
|
||
return false, "drive " + hdd + " is decommissioned"
|
||
}
|
||
if sp.Disconnected {
|
||
return false, "drive " + hdd + " is flagged disconnected"
|
||
}
|
||
}
|
||
}
|
||
if !g.mgr.DriveLive(hdd) {
|
||
return false, "drive " + hdd + " is not a live mountpoint"
|
||
}
|
||
return true, ""
|
||
}
|
||
|
||
// gatedAppStopStarter is the app-stop guard's starter, wrapped in the drive gate (R-174).
|
||
//
|
||
// THE DEFECT IT CLOSES, found by review on 2026-08-02 in code shipped 2026-08-01 (v0.189.0):
|
||
// `appStopGuard.SetStarter(stackMgr)` handed Recover the RAW stack manager, whose `StartStack` has
|
||
// no drive gate. Recover runs at startup — precisely when an external drive may not have come back —
|
||
// so a backup that stopped an app, followed by a power cut and a drive that did not remount, ended
|
||
// with the app started onto a missing drive. That is R-171 one path over, and the rule is not new:
|
||
// the API's own `startGatedByMissingDrive` already refuses this to the customer.
|
||
//
|
||
// The check stays HERE, per-caller, and is deliberately NOT pushed into `Manager.StartStack` — it has
|
||
// fourteen callers and most of them legitimately start apps outside this concern.
|
||
type gatedAppStopStarter struct {
|
||
inner backup.AppStopStarter
|
||
gate driveStartGate
|
||
logger *log.Logger
|
||
}
|
||
|
||
func (s gatedAppStopStarter) StartStack(name string) error {
|
||
if ok, why := s.gate.MayStart(name); !ok {
|
||
s.logger.Printf("[WARN] [appstop] refusing to restart %q after an interrupted operation: %s", name, why)
|
||
return fmt.Errorf("%w: %s", backup.ErrStartRefused, why)
|
||
}
|
||
return s.inner.StartStack(name)
|
||
}
|
||
|
||
// fillWatchStartupDelay is how long the startup fill check waits before its single run. Long enough
|
||
// for the drive gate's first reconcile and for mounts to settle — a drive still coming back must read
|
||
// as unreadable (and be skipped, §8.4), never as a filesystem worth warning about.
|
||
const fillWatchStartupDelay = 90 * time.Second
|
||
|
||
// fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every
|
||
// registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned
|
||
// between checks is picked up without a controller restart.
|
||
//
|
||
// WHY THESE THREE AND NOT ONLY THE REGISTERED DRIVES — which is all the healthcheck ever looked at:
|
||
//
|
||
// - the APP-DATA volume (`mp0`, `/var/lib/docker`) holds every app's live named volumes;
|
||
// - the SYSTEM-DATA volume (`mp1`, `/mnt/sys_drive`) holds the retained recovery unit of every
|
||
// DRIVELESS app (architecture/07-backup-architecture.md §7.5) and is the smaller of the two at
|
||
// 20 G against 50 G — the very mismatch R-165 exists to remove. It is also the filesystem whose
|
||
// silent exhaustion R-158 measured;
|
||
// - the registered DRIVES hold the customer's own data.
|
||
//
|
||
// De-duplicated by path: on a box where a drive is not a separate mount these collapse, and warning
|
||
// twice about one filesystem is exactly the noise the per-filesystem rule exists to prevent. A
|
||
// DECOMMISSIONED drive is dropped — it is deliberately out of service, and its fill is not news. A
|
||
// DISCONNECTED one is kept in the list but will read as unreadable and be skipped by §8.4, which is
|
||
// the correct outcome: its absence already has its own alert.
|
||
func fillTargets(cfg *config.Config, sett *settings.Settings) []fillwatch.Target {
|
||
seen := make(map[string]bool)
|
||
var out []fillwatch.Target
|
||
add := func(path, label string) {
|
||
if path == "" || seen[path] {
|
||
return
|
||
}
|
||
seen[path] = true
|
||
out = append(out, fillwatch.Target{Path: path, Label: label})
|
||
}
|
||
|
||
// The docker data-root as seen from inside the guest, and the system-data area.
|
||
add(system.DockerVolumePath, "Alkalmazások területe")
|
||
if cfg != nil {
|
||
add(cfg.Paths.SystemDataPath, "Rendszer- és mentési terület")
|
||
}
|
||
if sett != nil {
|
||
for _, sp := range sett.GetStoragePaths() {
|
||
if sp.Decommissioned {
|
||
continue
|
||
}
|
||
add(sp.Path, sp.Label)
|
||
}
|
||
}
|
||
return fillwatch.SortTargets(out)
|
||
}
|
||
|
||
// bootFleetSample is a comparable snapshot of one deployed app — name, state and container count,
|
||
// per §8.1. Container count is in it deliberately: a stack can go from 3 containers to 0 without its
|
||
// aggregate state changing, and that IS the boot still moving.
|
||
type bootFleetSample struct {
|
||
name string
|
||
state string
|
||
containers int
|
||
}
|
||
|
||
// sampleBootFleet REFRESHES, then returns the fleet snapshot, sorted, so two samples compare by
|
||
// equality.
|
||
//
|
||
// THE REFRESH IS LOAD-BEARING, and it was found by live validation, not by review. `GetStacks()`
|
||
// returns the Manager's IN-MEMORY map, which the scheduler refreshes on its own 10 s cadence
|
||
// (`status-refresh`, main.go). Sampling it every 5 s without refreshing means two consecutive samples
|
||
// can straddle one refresh and be identical because THE CACHE DID NOT UPDATE — not because the fleet
|
||
// stopped moving. The window would then declare "settled" on stale data and sweep on a picture of the
|
||
// box from up to 10 s ago, which is a quieter version of the exact defect R-157 mechanism A is.
|
||
//
|
||
// Observed on guest 9201 on 2026-08-02: a container removed ~5 s before the window closed was still
|
||
// present in the sampled fleet, so the sweep logged "no boot-orphaned apps" for an app that had none.
|
||
//
|
||
// `RefreshStatus` is a cheap `docker ps`-based refresh of the in-memory map (the same one the
|
||
// scheduler runs every 10 s), so at most ~10 extra calls per boot. A refresh error is logged and the
|
||
// sample proceeds on whatever the map holds — degraded, but never silently: a boot window that cannot
|
||
// see docker is exactly when a stale verdict is most dangerous.
|
||
func sampleBootFleet(mgr bootrecon.StackProvider) []bootFleetSample {
|
||
_ = mgr.RefreshStatus()
|
||
stacksNow := mgr.GetStacks()
|
||
out := make([]bootFleetSample, 0, len(stacksNow))
|
||
for _, s := range stacksNow {
|
||
if !s.Deployed {
|
||
continue
|
||
}
|
||
out = append(out, bootFleetSample{name: s.Name, state: string(s.State), containers: len(s.Containers)})
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return out[i].name < out[j].name })
|
||
return out
|
||
}
|
||
|
||
func sameBootFleet(a, b []bootFleetSample) bool {
|
||
if len(a) != len(b) {
|
||
return false
|
||
}
|
||
for i := range a {
|
||
if a[i] != b[i] {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// runBootReconcile waits out the settle delay, then samples the fleet until it stops changing (or
|
||
// the budget runs out) and performs the bounded sweep ONCE, at the end.
|
||
//
|
||
// Sweeping on every sample was rejected: the sweep's own StartStack changes the fleet, so a
|
||
// sweep-per-sample would never observe a settled fleet and would race docker's restore. Sampling is
|
||
// read-only; exactly one sweep runs, and it re-derives its candidate set from a settled fleet —
|
||
// which is the whole point, since the pre-v0.190.0 bug was a candidate set derived too early.
|
||
//
|
||
// Called from main() in a goroutine.
|
||
func runBootReconcile(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(bootReconcileSettle):
|
||
}
|
||
|
||
started := time.Now()
|
||
prev := sampleBootFleet(mgr)
|
||
stable := 1
|
||
settled := false
|
||
|
||
for time.Since(started) < bootReconcileBudget {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(bootReconcileSample):
|
||
}
|
||
cur := sampleBootFleet(mgr)
|
||
if sameBootFleet(prev, cur) {
|
||
stable++
|
||
} else {
|
||
// Not settled — the boot is still moving. Log at DEBUG: at 5 s cadence an INFO line per
|
||
// sample would bury the one line that matters, which is the verdict below.
|
||
logger.Printf("[DEBUG] [bootrecon] boot window: fleet still changing (%d app(s)) — resampling", len(cur))
|
||
stable = 1
|
||
}
|
||
prev = cur
|
||
if stable >= bootReconcileStableFor {
|
||
settled = true
|
||
break
|
||
}
|
||
}
|
||
|
||
if settled {
|
||
logger.Printf("[INFO] [bootrecon] boot window: fleet settled after %.0fs (%d identical samples %s apart) — sweeping",
|
||
time.Since(started).Seconds(), bootReconcileStableFor, bootReconcileSample)
|
||
} else {
|
||
// NOT a failure — a box whose apps are still churning at the budget is exactly the box that
|
||
// most needs the sweep. But it is a different fact from "settled", and saying so is what makes
|
||
// a stuck boot visible instead of looking like a quiet one.
|
||
logger.Printf("[INFO] [bootrecon] boot window: budget %s exhausted while the fleet was still changing — sweeping anyway",
|
||
bootReconcileBudget)
|
||
}
|
||
|
||
res := bootReconcileFn(ctx, mgr, logger)
|
||
recordLateRecovery(logger, started, res)
|
||
}
|
||
|
||
// recordLateRecovery keeps §8.3 honest. The window can finish AFTER deadAppBootGrace, and when it
|
||
// does the dead-app alarm has already fired for an app this sweep then recovered. Extending the
|
||
// grace to hide that was rejected; reporting it is the alternative, so the record is truthful and
|
||
// the operator is not left with a stale alarm and no counter-evidence.
|
||
//
|
||
// Measured from controller start, which is what the grace is measured from.
|
||
func recordLateRecovery(logger *log.Logger, started time.Time, res bootrecon.Result) {
|
||
if len(res.Recovered) == 0 {
|
||
return
|
||
}
|
||
elapsed := bootReconcileSettle + time.Since(started)
|
||
if elapsed <= deadAppBootGrace {
|
||
return
|
||
}
|
||
logger.Printf("[WARN] [bootrecon] LATE RECOVERY: %d app(s) recovered %.0fs after start, past the %s dead-app grace — an alert may already have fired for: %v",
|
||
len(res.Recovered), elapsed.Seconds(), deadAppBootGrace, res.Recovered)
|
||
}
|
||
|
||
// scanDeployedAppRunStates returns the fix-3 view of the deployed apps: the DEAD ones (for the
|
||
// state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
|
||
// transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks()
|
||
// — the derivation itself lives in classifyRunStates so it is testable without a live Manager.
|
||
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop) ([]web.DeadApp, []notify.AppRunState) {
|
||
// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
|
||
// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
|
||
return classifyRunStates(mgr.GetStacks(), q.SuppressedStacks(), q.FailedRestarts(), time.Now())
|
||
}
|
||
|
||
// classifyRunStates is the pure fix-3 derivation over a plain stack slice. It splits the deployed
|
||
// apps into the DEAD list (dashboard banner) and the per-app run states (notifier transition tracker).
|
||
//
|
||
// v0.164.0: a deliberate user stop is NOT a fault and must not alarm anywhere (banner OR email). The
|
||
// down predicate therefore excludes StateStopped — but NOT unconditionally. That exclusion rested on
|
||
// two invariants, and F-CRIT-1 (Campaign 8) showed the first of them had already become false:
|
||
//
|
||
// - I1 (AS WRITTEN, AND WRONG): "the UI stop path Manager.StopStack runs `docker compose down` →
|
||
// containers are removed → a deployed stack with zero containers aggregates to StateStopped, so
|
||
// StateStopped means 'deployed, deliberately stopped by the user'."
|
||
// **The quiesce loop stops stacks by the SAME path.** So a stack that a backup quiesce stopped and
|
||
// then FAILED to restart is also StateStopped — byte-identical to a user stop on the Docker side —
|
||
// and the whitelist swallowed it. Campaign 8 observed a customer app dead indefinitely with no
|
||
// banner, no event and no email, while the dead-app scanner ran 11 times over it.
|
||
// - I2 (still true): the P2 restart-policy census (2026-07-21, 53 templates / 78 services) found
|
||
// every catalog service on `unless-stopped`, so a CRASHING app never comes to rest at `stopped` —
|
||
// faults surface as StateExited / StateDegraded (and restarting/unhealthy).
|
||
//
|
||
// I1 RESTATED, TRUE AS OF v0.179.0: StateStopped means "deployed with zero containers", which is
|
||
// EITHER a deliberate user stop OR a quiesce restart that failed. No state test can tell them apart,
|
||
// because they are the same state. The distinguishing fact is not in the state at all — it is that
|
||
// the quiesce loop TRIED to restart the stack and could not, which it now reports via
|
||
// Loop.FailedRestarts(). `failedRestart` is that set, and it is the ONLY thing that lifts the
|
||
// StateStopped whitelist.
|
||
//
|
||
// (An out-of-band `docker compose stop` leaves the containers present → StateExited → still alerts,
|
||
// which is correct: out-of-band tampering IS reportable.) IsDownState is intentionally left unchanged
|
||
// — other callers rely on stopped counting as down; the suppression is a filter at this single
|
||
// derivation point only.
|
||
// `now` is injected (C9-F2) so the crash-loop threshold is a unit-testable contract rather than a
|
||
// property of the wall clock.
|
||
func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool, failedRestart map[string]bool, now time.Time) ([]web.DeadApp, []notify.AppRunState) {
|
||
var dead []web.DeadApp
|
||
var states []notify.AppRunState
|
||
for _, st := range sts {
|
||
if !st.Deployed || st.Deploying {
|
||
continue
|
||
}
|
||
// R-97b: a stack a quiesce cycle stopped (or restarted within the grace window) is NOT down —
|
||
// we stopped it. This is a CYCLE-keyed suppression, not a state test, because an app caught
|
||
// mid-restart is `starting`/`unhealthy`, not StateStopped, so v0.164.0's state filter above
|
||
// cannot see it. The window EXPIRES (quiesceAlarmGrace): an app that genuinely fails to come
|
||
// back still alarms on the first scan after it closes.
|
||
// F-CRIT-1: StateStopped is whitelisted as a deliberate user stop UNLESS the quiesce loop
|
||
// reports that it stopped this stack and could not restart it. That single term is what turns
|
||
// an indefinitely-silent dead app back into an alarm, without re-alarming genuine user stops.
|
||
// C9-F2: a SUSTAINED restarting is a crash loop, and a crash loop is a dead app. Deliberately
|
||
// NOT folded into IsDownState — that would alarm on every deploy and update fleet-wide, which
|
||
// is the over-correction F-A1 nearly cost us. The threshold (stacks.crashLoopAfter, 5 min) sits
|
||
// above the deploy health timeout, the slowest catalog start_period AND R-97b's grace, so a
|
||
// brief restart never reaches it and the two suppression windows compose into one bounded
|
||
// delay. Quiesce suppression below still wins inside its own window.
|
||
crashLooping := st.CrashLooping(now)
|
||
userStopped := st.State == stacks.StateStopped && !failedRestart[st.Name]
|
||
down := (stacks.IsDownState(st.State) || crashLooping) && !userStopped && !quiesced[st.Name]
|
||
states = append(states, notify.AppRunState{Name: st.Name, DisplayName: st.Meta.DisplayName, Down: down})
|
||
if down {
|
||
dead = append(dead, web.DeadApp{Name: st.Name, DisplayName: st.Meta.DisplayName, State: string(st.State)})
|
||
}
|
||
}
|
||
return dead, states
|
||
}
|
||
|
||
func selfUpdateAuthMiddleware(cfg *config.Config, webServer *web.Server, next http.Handler) http.Handler {
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
// Check bearer token first (for external API calls: hub, build scripts)
|
||
if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "Bearer ") {
|
||
token := strings.TrimPrefix(auth, "Bearer ")
|
||
if token != "" && cfg.Hub.APIKey != "" && subtle.ConstantTimeCompare([]byte(token), []byte(cfg.Hub.APIKey)) == 1 {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
}
|
||
// Fall back to session auth
|
||
webServer.RequireAuth(next).ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// setupLogger builds the v0.116.0 capture layer: the LogBuffer ring ALWAYS exists
|
||
// and captures every line (DEBUG included — remote diagnostics need the detail to
|
||
// exist without a config flip), while stdout (docker logs) keeps respecting
|
||
// logging.level via the level filter. Lshortfile stays debug-only (unchanged).
|
||
func setupLogger(cfg *config.Config) (*log.Logger, *web.LogBuffer) {
|
||
// fix-6 (CAMPAIGN-3): 1000 wrapped in ~6.5 min under the campaign's ~2.6 entries/s load — the exact
|
||
// post-incident window an operator needs was the first thing lost. 5000 gives ≈32 min at that raw
|
||
// rate, and ≈50+ min now that the periodic-scheduler TRACE noise no longer enters the ring (2b).
|
||
logBuffer := web.NewLogBuffer(5000)
|
||
flags := log.LstdFlags
|
||
if cfg.Logging.Level == "debug" {
|
||
flags |= log.Lshortfile
|
||
}
|
||
stdout := web.NewLevelFilterWriter(os.Stdout, cfg.Logging.Level)
|
||
return log.New(io.MultiWriter(stdout, logBuffer), "", flags), logBuffer
|
||
}
|
||
|
||
// stackAdapter implements backup.StackDataProvider using stacks.Manager.
|
||
type stackAdapter struct {
|
||
mgr *stacks.Manager
|
||
getStoragePaths func() []settings.StoragePath
|
||
encKey []byte // for decrypting live app.yaml secrets during restore-from-unit
|
||
}
|
||
|
||
func (a *stackAdapter) GetStackComposePath(name string) (string, bool) {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
return s.ComposePath, true
|
||
}
|
||
|
||
// GetStackClassifiedBinds delegates to the stacks manager's backup-classification resolver (Task 2).
|
||
// INERT — no backup tier calls this yet; wired so Task 3 consumes a tested seam.
|
||
func (a *stackAdapter) GetStackClassifiedBinds(name string) ([]backup.ClassifiedBind, bool) {
|
||
return a.mgr.ClassifiedBinds(name)
|
||
}
|
||
|
||
func (a *stackAdapter) ListDeployedStacks() []backup.StackSummary {
|
||
var result []backup.StackSummary
|
||
for _, s := range a.mgr.GetStacks() {
|
||
if !s.Deployed {
|
||
continue
|
||
}
|
||
result = append(result, backup.StackSummary{
|
||
Name: s.Name,
|
||
DisplayName: s.Meta.DisplayName,
|
||
ComposePath: s.ComposePath,
|
||
NeedsHDD: s.Meta.Resources.NeedsHDD,
|
||
HasVolumes: len(backup.ParseComposeNamedVolumes(s.ComposePath)) > 0,
|
||
})
|
||
}
|
||
return result
|
||
}
|
||
|
||
func (a *stackAdapter) StopStack(name string) error {
|
||
return a.mgr.StopStack(name)
|
||
}
|
||
|
||
func (a *stackAdapter) StartStack(name string) error {
|
||
return a.mgr.StartStack(name)
|
||
}
|
||
|
||
func (a *stackAdapter) GetStackHDDMounts(name string) []string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
|
||
// Priority 1: Read the app's own HDD_PATH from its app.yaml
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
appCfg := stacks.LoadAppConfig(stackDir)
|
||
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
|
||
return stacks.ParseComposeHDDMounts(s.ComposePath, appCfg.Env["HDD_PATH"])
|
||
}
|
||
|
||
// Priority 2: Try all registered storage paths (fallback)
|
||
var allMounts []string
|
||
seen := make(map[string]bool)
|
||
for _, sp := range a.getStoragePaths() {
|
||
mounts := stacks.ParseComposeHDDMounts(s.ComposePath, sp.Path)
|
||
for _, m := range mounts {
|
||
if !seen[m] {
|
||
seen[m] = true
|
||
allMounts = append(allMounts, m)
|
||
}
|
||
}
|
||
}
|
||
return allMounts
|
||
}
|
||
|
||
func (a *stackAdapter) GetDockerVolumes(name string) []string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
return backup.ResolveDockerVolumeNames(s.ComposePath)
|
||
}
|
||
|
||
// GetImportRoot delegates to the stack manager's canonical drop-zone root (R-75). Both the backup
|
||
// and the export provider need it: ${IMPORT_PATH} binds live on the SYSTEM drive and can NOT be
|
||
// resolved from an app's HDD_PATH.
|
||
func (a *stackAdapter) GetImportRoot() string { return a.mgr.GetImportRoot() }
|
||
|
||
func (a *stackAdapter) GetStackHDDPath(name string) string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return ""
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
appCfg := stacks.LoadAppConfig(stackDir)
|
||
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
|
||
return filepath.Clean(appCfg.Env["HDD_PATH"])
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// GetStackRecoveryInfo gathers the inputs for an app's recovery unit: the stack dir, pinned image
|
||
// tags, the non-secret env, the NAMES of every secret/data-key env var, and — since D5 — the decrypted
|
||
// VALUES of the PORTABLE secret class.
|
||
//
|
||
// The non-secret env still excludes every named secret (plus a defensive crypto.IsEncrypted guard), so
|
||
// the two sets are disjoint by construction and a secret can only reach the unit by being in the
|
||
// portable class. The EXCLUDED class (`type: password`, and the nonPortableSecrets register) is
|
||
// name-only here and is recovered from the guest — or regenerated (O4) — exactly as before.
|
||
func (a *stackAdapter) GetStackRecoveryInfo(name string) (backup.RecoveryInfo, bool) {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return backup.RecoveryInfo{}, false
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
meta := stacks.LoadMetadata(stackDir)
|
||
|
||
// Secret set = all secret/password fields ∪ any data_key fields (in deterministic metadata order).
|
||
secretSet := make(map[string]bool)
|
||
var secretNames []string
|
||
add := func(v string) {
|
||
if !secretSet[v] {
|
||
secretSet[v] = true
|
||
secretNames = append(secretNames, v)
|
||
}
|
||
}
|
||
for _, v := range stacks.SensitiveEnvVars(&meta) {
|
||
add(v)
|
||
}
|
||
dataKeys := meta.DataKeyEnvVars()
|
||
for _, v := range dataKeys {
|
||
add(v)
|
||
}
|
||
|
||
// Non-secret env: raw app.yaml values that are neither named-secret nor (defensively) encrypted.
|
||
nonSecret := make(map[string]string)
|
||
if appCfg := stacks.LoadAppConfig(stackDir); appCfg != nil {
|
||
for k, v := range appCfg.Env {
|
||
if secretSet[k] || crypto.IsEncrypted(v) {
|
||
continue
|
||
}
|
||
nonSecret[k] = v
|
||
}
|
||
}
|
||
|
||
// D5: decrypt the portable class so it can travel in the unit. Reuses the SAME decrypt path as the
|
||
// restore side (LoadAppConfigDecrypted), so there is one way to turn app.yaml into plaintext, not
|
||
// two. A name whose value is absent/empty is simply omitted — the restore's fail-closed data-key
|
||
// gate is what decides whether that is survivable.
|
||
portableNames := stacks.PortableSecretEnvVars(&meta)
|
||
portable := make(map[string]string, len(portableNames))
|
||
if len(portableNames) > 0 {
|
||
if dec := stacks.LoadAppConfigDecrypted(stackDir, a.encKey); dec != nil {
|
||
for _, n := range portableNames {
|
||
if v, ok := dec.Env[n]; ok && v != "" {
|
||
portable[n] = v
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return backup.RecoveryInfo{
|
||
StackDir: stackDir,
|
||
DisplayName: s.Meta.DisplayName,
|
||
ImagePins: backup.ParseComposeImages(s.ComposePath),
|
||
NonSecretEnv: nonSecret,
|
||
SecretEnvVars: secretNames,
|
||
DataKeyEnvVars: dataKeys,
|
||
PortableSecretEnvVars: portableNames,
|
||
PortableSecrets: portable,
|
||
}, true
|
||
}
|
||
|
||
// RecoverStackSecrets returns the live decrypted values for the named secret env vars present in the
|
||
// stack's app.yaml (the guest's own — live rootfs or PBS-restored). Absent/empty names are omitted;
|
||
// the caller's fail-closed gate decides. Secrets come from the guest, never from the recovery unit.
|
||
func (a *stackAdapter) RecoverStackSecrets(name string, names []string) map[string]string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
cfg := stacks.LoadAppConfigDecrypted(filepath.Dir(s.ComposePath), a.encKey)
|
||
if cfg == nil {
|
||
return nil
|
||
}
|
||
out := make(map[string]string)
|
||
for _, n := range names {
|
||
if v, ok := cfg.Env[n]; ok && v != "" {
|
||
out[n] = v
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// RecreateStackDefinitionFromUnit restores the app definition from the unit's compose dir into the
|
||
// stack dir and persists app.yaml from the reconstructed full env. Secrets in fullEnv were recovered
|
||
// from the guest, never regenerated. It starts NOTHING — the restore flow brings the database service
|
||
// up alone for the dump replay and only then starts the whole stack (R-47).
|
||
func (a *stackAdapter) RecreateStackDefinitionFromUnit(name, composeSrcDir string, fullEnv map[string]string) error {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return fmt.Errorf("stack %q not found", name)
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
// Recover the app definition from the unit (compose + .felhom.yml) into the stack dir.
|
||
for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} {
|
||
data, err := os.ReadFile(filepath.Join(composeSrcDir, fname))
|
||
if err != nil {
|
||
continue // capture whichever existed
|
||
}
|
||
if err := os.WriteFile(filepath.Join(stackDir, fname), data, 0644); err != nil {
|
||
return fmt.Errorf("restoring %s from unit: %w", fname, err)
|
||
}
|
||
}
|
||
return a.mgr.PersistUnitRedeployConfig(name, fullEnv)
|
||
}
|
||
|
||
// StartStackServices brings up only the named compose services (the DB-only replay window, R-47).
|
||
func (a *stackAdapter) StartStackServices(name string, services []string) error {
|
||
return a.mgr.StartStackServices(name, services)
|
||
}
|
||
|
||
// RefreshAndIsRunning forces a docker ps scan before checking state.
|
||
// Called during post-restore health check (~every 5s for up to 90s).
|
||
// Full refresh is acceptable here since restores are rare operations.
|
||
func (a *stackAdapter) RefreshAndIsRunning(name string) bool {
|
||
a.mgr.RefreshStatus()
|
||
s, ok := a.mgr.GetStack(name)
|
||
return ok && s.State == stacks.StateRunning
|
||
}
|
||
|
||
// integrationStackAdapter implements integrations.StackProvider using stacks.Manager.
|
||
type integrationStackAdapter struct {
|
||
mgr *stacks.Manager
|
||
}
|
||
|
||
func (a *integrationStackAdapter) GetStack(name string) (*stacks.Stack, bool) {
|
||
return a.mgr.GetStack(name)
|
||
}
|
||
|
||
func (a *integrationStackAdapter) GetStacks() []stacks.Stack {
|
||
return a.mgr.GetStacks()
|
||
}
|
||
|
||
func (a *integrationStackAdapter) RestartStack(name string) error {
|
||
return a.mgr.RestartStack(name)
|
||
}
|
||
|
||
// geoStackAdapter implements cloudflare.StackLister for geo-restriction sync.
|
||
type geoStackAdapter struct {
|
||
mgr *stacks.Manager
|
||
domain string
|
||
}
|
||
|
||
func (a *geoStackAdapter) GetDeployedHostnames() map[string]string {
|
||
result := make(map[string]string)
|
||
for _, stack := range a.mgr.GetStacks() {
|
||
if !stack.Deployed {
|
||
continue
|
||
}
|
||
subdomain := stack.Meta.Subdomain
|
||
// Check for custom subdomain in app.yaml
|
||
if appCfg := a.mgr.LoadAppConfigByName(stack.Name); appCfg != nil {
|
||
if sd, ok := appCfg.Env["SUBDOMAIN"]; ok && sd != "" {
|
||
subdomain = sd
|
||
}
|
||
}
|
||
if subdomain != "" {
|
||
result[stack.Name] = subdomain + "." + a.domain
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// exportAdapter implements appexport.ExportStackProvider using stacks.Manager.
|
||
type exportAdapter struct {
|
||
mgr *stacks.Manager
|
||
encKey []byte
|
||
}
|
||
|
||
func (a *exportAdapter) GetStackDir(name string) (string, bool) {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
return filepath.Dir(s.ComposePath), true
|
||
}
|
||
|
||
// GetStackClassifiedBinds delegates to the stacks manager's backup-classification resolver (Task 2),
|
||
// used by the `.fab` class-scoped export plan (Task 4).
|
||
func (a *exportAdapter) GetStackClassifiedBinds(name string) ([]appbackup.ClassifiedBind, bool) {
|
||
return a.mgr.ClassifiedBinds(name)
|
||
}
|
||
|
||
func (a *exportAdapter) GetStackComposePath(name string) (string, bool) {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
return s.ComposePath, true
|
||
}
|
||
|
||
func (a *exportAdapter) GetStackHDDMounts(name string) []string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
appCfg := stacks.LoadAppConfig(stackDir)
|
||
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
|
||
// C6B-F1 (v0.130.0): union ${HDD_PATH} binds + the ${USERDATA_PATH} root. The old
|
||
// ParseComposeHDDMounts-only call was blind to the standard userdata convention, so
|
||
// 12/13 needs_hdd catalog apps exported hollow (config-only) bundles. The backup-side
|
||
// stackAdapter is intentionally NOT changed here — the scheduled/tier-2 path copies the
|
||
// recovery unit + the app's resolved appdata/<name> dir(s) only (NOT the userdata tree —
|
||
// F-S1; NOT the namespace wholesale), and derives that dir name from the compose binds
|
||
// (F-S2, see backup.tier2AppDataName). Spec:
|
||
// felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md.
|
||
return stacks.ExportDataMounts(s.ComposePath, appCfg.Env["HDD_PATH"], a.mgr.StackNamespaceRoot(name))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// GetImportRoot delegates to the stack manager's canonical drop-zone root (R-75). Both the backup
|
||
// and the export provider need it: ${IMPORT_PATH} binds live on the SYSTEM drive and can NOT be
|
||
// resolved from an app's HDD_PATH.
|
||
func (a *exportAdapter) GetImportRoot() string { return a.mgr.GetImportRoot() }
|
||
|
||
// GetStackNamespaceRoot (R-203) — the felhom-data namespace root, NOT the drive path. The appbackup
|
||
// path helpers all take this; passing HDD_PATH straight in is what made the export plan and the
|
||
// off-site capture set describe different directories on the system-data fallback.
|
||
func (a *exportAdapter) GetStackNamespaceRoot(name string) string { return a.mgr.StackNamespaceRoot(name) }
|
||
|
||
func (a *exportAdapter) GetStackHDDPath(name string) string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return ""
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
appCfg := stacks.LoadAppConfig(stackDir)
|
||
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
|
||
return filepath.Clean(appCfg.Env["HDD_PATH"])
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (a *exportAdapter) IsStackRunning(name string) bool {
|
||
s, ok := a.mgr.GetStack(name)
|
||
// StateDegraded (R-51) counts as running: the export must stop the still-live members before
|
||
// reading their volumes, exactly as it would for a fully running stack.
|
||
return ok && (s.State == stacks.StateRunning || s.State == stacks.StateDegraded)
|
||
}
|
||
|
||
func (a *exportAdapter) StopStack(name string) error {
|
||
return a.mgr.StopStack(name)
|
||
}
|
||
|
||
func (a *exportAdapter) StartStack(name string) error {
|
||
return a.mgr.StartStack(name)
|
||
}
|
||
|
||
func (a *exportAdapter) GetStackDisplayName(name string) string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return name
|
||
}
|
||
return s.Meta.DisplayName
|
||
}
|
||
|
||
func (a *exportAdapter) GetStackNeedsHDD(name string) bool {
|
||
s, ok := a.mgr.GetStack(name)
|
||
return ok && s.Meta.Resources.NeedsHDD
|
||
}
|
||
|
||
func (a *exportAdapter) GetDockerVolumes(name string) []string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
return backup.ResolveDockerVolumeNames(s.ComposePath)
|
||
}
|
||
|
||
func (a *exportAdapter) IsStackDeployed(name string) bool {
|
||
s, ok := a.mgr.GetStack(name)
|
||
return ok && s.Deployed
|
||
}
|
||
|
||
func (a *exportAdapter) GetDecryptedEnv(name string) map[string]string {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
cfg := stacks.LoadAppConfigDecrypted(stackDir, a.encKey)
|
||
if cfg == nil {
|
||
return nil
|
||
}
|
||
return cfg.Env
|
||
}
|
||
|
||
func (a *exportAdapter) GetStacksBaseDir() string {
|
||
return a.mgr.GetStacksBaseDir()
|
||
}
|
||
|
||
// exportStopGuard adapts *backup.AppStopGuard to the exporter's reason-free seam (R-166). The reason
|
||
// is supplied HERE rather than passed in, so backup.ReasonAppExport's value exists in exactly one
|
||
// place and the two packages cannot drift apart.
|
||
type exportStopGuard struct{ g *backup.AppStopGuard }
|
||
|
||
func (a exportStopGuard) Begin(opID string, stacks []string) error {
|
||
return a.g.Begin(opID, backup.ReasonAppExport, stacks)
|
||
}
|
||
|
||
func (a exportStopGuard) End() { a.g.End() }
|
||
|
||
func (a *exportAdapter) SaveEncryptedAppConfig(stackDir string, env map[string]string) error {
|
||
meta := stacks.LoadMetadata(stackDir)
|
||
sensitiveVars := stacks.SensitiveEnvVars(&meta)
|
||
cfg := &stacks.AppConfig{
|
||
Deployed: true,
|
||
DeployedAt: time.Now().Format(time.RFC3339),
|
||
Env: env,
|
||
// R-166 — a CUSTOMER-INTENT POINT, and the one that is not the API action switch. Importing
|
||
// a `.fab` bundle is the customer installing that app on this box, and the import path starts
|
||
// it (appexport/restore.go). Without this the app would come back from a restore with NO
|
||
// recorded intent and fall to legacy boot behaviour — meaning a power cut days later would
|
||
// strand it, which is exactly the failure this release exists to remove.
|
||
DesiredState: stacks.DesiredStateRunning,
|
||
}
|
||
return stacks.SaveAppConfig(stackDir, cfg, a.encKey, sensitiveVars)
|
||
}
|
||
|
||
func (a *exportAdapter) RefreshStacks() error {
|
||
return a.mgr.RefreshStatus()
|
||
}
|
||
|
||
func (a *exportAdapter) RemoveStackVolumes(name string) error {
|
||
s, ok := a.mgr.GetStack(name)
|
||
if !ok {
|
||
return fmt.Errorf("stack %q not found", name)
|
||
}
|
||
stackDir := filepath.Dir(s.ComposePath)
|
||
|
||
// Build env from decrypted app config
|
||
cmdEnv := os.Environ()
|
||
appCfg := stacks.LoadAppConfigDecrypted(stackDir, a.encKey)
|
||
if appCfg != nil {
|
||
for k, v := range appCfg.Env {
|
||
cmdEnv = append(cmdEnv, k+"="+v)
|
||
}
|
||
}
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||
defer cancel()
|
||
cmd := exec.CommandContext(ctx, "docker", "compose", "down", "--volumes")
|
||
cmd.Dir = stackDir
|
||
cmd.Env = cmdEnv
|
||
out, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
return fmt.Errorf("compose down --volumes: %s — %w", strings.TrimSpace(string(out)), err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// fileExists returns true if the path exists (file or directory).
|
||
func fileExists(path string) bool {
|
||
_, err := os.Stat(path)
|
||
return err == nil
|
||
}
|
||
|
||
// quiesceBackend adapts *agentapi.Client to quiesce.Backend (bool/string, decoupled from the
|
||
// agentapi response structs).
|
||
type quiesceBackend struct{ c *agentapi.Client }
|
||
|
||
func (b quiesceBackend) Due(ctx context.Context) (bool, *int64, error) {
|
||
r, err := b.c.BackupDue(ctx)
|
||
return r.Due, r.AgeSecs, err
|
||
}
|
||
func (b quiesceBackend) StartBackup(ctx context.Context) (string, error) {
|
||
r, err := b.c.StartBackup(ctx)
|
||
return r.JobID, mapBusy(err) // F-A1: the untargeted (pre-R-82) path can 409 identically
|
||
}
|
||
func (b quiesceBackend) BackupStatus(ctx context.Context) (string, error) {
|
||
r, err := b.c.BackupStatus(ctx)
|
||
return r.Phase, err
|
||
}
|
||
|
||
// ---- R-82: the tiered surface (quiesce.TieredBackend) ------------------------------------
|
||
//
|
||
// quiesceBackend satisfies quiesce.TieredBackend as well, so the loop schedules per tier when the
|
||
// agent supports it. Against a PRE-R-82 agent, Tiers returns quiesce.ErrTiersUnsupported and the
|
||
// loop degrades to the untargeted methods above — still taking a backup, never skipping one.
|
||
|
||
func (b quiesceBackend) Tiers(ctx context.Context) ([]quiesce.BackupTier, error) {
|
||
r, err := b.c.BackupTiers(ctx)
|
||
if errors.Is(err, agentapi.ErrTiersUnsupported) {
|
||
// Translate the transport-layer probe into the loop's vocabulary; the loop keys on this.
|
||
return nil, quiesce.ErrTiersUnsupported
|
||
}
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]quiesce.BackupTier, 0, len(r.Tiers))
|
||
for _, t := range r.Tiers {
|
||
out = append(out, quiesce.BackupTier{Target: t.Target, Primary: t.Primary})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64, string, error) {
|
||
r, err := b.c.BackupDueFor(ctx, target)
|
||
// AgeState is passed through RAW; quiesce.ageStateFromWire owns the mapping, including the
|
||
// legacy-vs-unknown distinction. An empty string here means a pre-v0.105.0 agent.
|
||
return r.Due, r.AgeSecs, r.AgeState, err
|
||
}
|
||
func (b quiesceBackend) StartBackupFor(ctx context.Context, target string) (string, error) {
|
||
r, err := b.c.StartBackupFor(ctx, target)
|
||
// F-A1: translate the agent's HTTP 409 into the loop's vocabulary, exactly as Tiers translates a
|
||
// 404 into ErrTiersUnsupported. 409 means the agent's R-85 single-flight gate refused because a
|
||
// restore-test (or another backup) holds it — CONTENTION, not failure. `internal/quiesce` keeps
|
||
// no dependency on `internal/agentapi`, so the mapping belongs here at the seam.
|
||
return r.JobID, mapBusy(err)
|
||
}
|
||
|
||
// mapBusy converts a 409 from the agent into quiesce.ErrTierBusy, preserving the original error for
|
||
// diagnosis. Any other status passes through untouched — treating a real 5xx as contention would
|
||
// swallow genuine failures, which is the over-correction F-A1's fix must not make.
|
||
func mapBusy(err error) error {
|
||
if err == nil {
|
||
return nil
|
||
}
|
||
var se *agentapi.StatusError
|
||
if errors.As(err, &se) && se.Code == http.StatusConflict {
|
||
return fmt.Errorf("%w: %v", quiesce.ErrTierBusy, err)
|
||
}
|
||
return err
|
||
}
|
||
func (b quiesceBackend) BackupStatusFor(ctx context.Context, target string) (string, error) {
|
||
r, err := b.c.BackupStatusFor(ctx, target)
|
||
return r.Phase, err
|
||
}
|
||
|
||
// quiesceTierNotifier adapts *notify.Notifier to quiesce.TierNotifier (R-97a).
|
||
//
|
||
// The whole-guest tier had NO route to the hub at all — `internal/quiesce` did not import
|
||
// `internal/notify`, so on 2026-07-27 three failed backups and twelve app-stack stop/starts produced
|
||
// ZERO `backup_failed` events. The event type was already in the hub's allowlist and
|
||
// `NotifyBackupFailed` already existed; only this adapter and its wiring were missing.
|
||
//
|
||
// The tier is carried in the MESSAGE rather than a new event type, because the hub gates on
|
||
// `allowedEventTypes` and a per-tier type would need a hub-side change to be deliverable at all.
|
||
// See the cooldown note in REPORT.md: the hub's operator cooldown is keyed
|
||
// `customerID + ":" + eventType`, so two tiers failing within an hour share one key — both events
|
||
// are STORED, but only the first sends an operator email.
|
||
type quiesceTierNotifier struct{ n *notify.Notifier }
|
||
|
||
func (q quiesceTierNotifier) BackupFailed(tier, message, errMsg string) {
|
||
q.n.NotifyWholeGuestBackupFailed(tier, message, errMsg)
|
||
}
|
||
|
||
func (q quiesceTierNotifier) BackupRecovered(tier, message string) {
|
||
q.n.NotifyWholeGuestBackupRecovered(tier, message)
|
||
}
|
||
|
||
// startQuiesceLoop wires + starts the slice-8B quiesce loop when the local API is configured and
|
||
// quiesce is enabled. It Recovers (restarts stacks left stopped by a mid-quiesce crash) before
|
||
// starting the loop goroutine. Non-fatal: any misconfig disables the loop with a log line.
|
||
func startQuiesceLoop(ctx context.Context, cfg *config.Config, sett *settings.Settings, stackMgr *stacks.Manager, logger *log.Logger) *quiesce.Loop {
|
||
if cfg.LocalAPI.Endpoint == "" || cfg.LocalAPI.Token == "" {
|
||
return nil // not a provisioned guest — no agent to back up against
|
||
}
|
||
if !cfg.Quiesce.QuiesceEnabled() {
|
||
logger.Printf("[INFO] [quiesce] disabled by config")
|
||
return nil
|
||
}
|
||
client, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
logger.Printf("[WARN] [quiesce] disabled (agent client init failed): %v", err)
|
||
return nil
|
||
}
|
||
poll := parseDurationOr(cfg.Quiesce.PollInterval, 5*time.Minute)
|
||
statusPoll := parseDurationOr(cfg.Quiesce.StatusPoll, 10*time.Second)
|
||
maxQuiesce := parseDurationOr(cfg.Quiesce.MaxQuiesce, 30*time.Minute)
|
||
loop := quiesce.New(quiesce.Options{
|
||
Backend: quiesceBackend{c: client},
|
||
Stacks: stackMgr,
|
||
MarkerPath: filepath.Join(cfg.Paths.DataDir, "quiesce-state.json"),
|
||
Poll: poll,
|
||
StatusPoll: statusPoll,
|
||
MaxQuiesce: maxQuiesce,
|
||
Logger: logger,
|
||
// Window gate (v0.168.0): read the effective window fresh each poll so a customer change takes
|
||
// effect without restart. Scheduled cycles run only inside [W+2h, W+6h), with the safety valve.
|
||
WindowStartFn: func() string {
|
||
return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
|
||
},
|
||
})
|
||
loop.Recover() // crash-safety: restart any stacks stranded-down by a mid-quiesce crash
|
||
go loop.Run(ctx)
|
||
return loop
|
||
}
|
||
|
||
// parseDurationOr parses a duration string, falling back to def on empty/invalid input.
|
||
func parseDurationOr(s string, def time.Duration) time.Duration {
|
||
if s == "" {
|
||
return def
|
||
}
|
||
d, err := time.ParseDuration(s)
|
||
if err != nil || d <= 0 {
|
||
return def
|
||
}
|
||
return d
|
||
}
|
||
|
||
// channelSink adapts the notify.Notifier + web.AlertManager to the channelhealth.Sink seam (kept in
|
||
// main.go so the channelhealth package imports neither — no cycle). SetDashboard reflects the current
|
||
// state each probe (idempotent); NotifyDown/NotifyRecovered fire only on a transition.
|
||
type channelSink struct {
|
||
notifier *notify.Notifier
|
||
alertMgr *web.AlertManager
|
||
}
|
||
|
||
func (s channelSink) SetDashboard(down bool, _ channelhealth.Reason, msg string) {
|
||
s.alertMgr.SetAgentChannelAlert(down, msg)
|
||
}
|
||
func (s channelSink) NotifyDown(reason channelhealth.Reason, eventType, severity, msg string) {
|
||
s.notifier.NotifyAgentChannelDown(string(reason), eventType, severity, msg)
|
||
}
|
||
func (s channelSink) NotifyRecovered() { s.notifier.NotifyAgentChannelRecovered() }
|
||
|
||
// probeLocalAPI proves the controller↔agent local-API channel at startup and logs this guest's
|
||
// mounts (slice 8A). Non-fatal: it only runs when a local-API endpoint is configured, and any
|
||
// error is logged for diagnosis without affecting the controller's boot. The leaf SHA-256 from
|
||
// the bootstrap is pinned by the client (fails closed on mismatch).
|
||
func probeLocalAPI(cfg *config.Config, logger *log.Logger) {
|
||
if cfg.LocalAPI.Endpoint == "" || cfg.LocalAPI.Token == "" {
|
||
return
|
||
}
|
||
client, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint)
|
||
if err != nil {
|
||
logger.Printf("[WARN] local-api: client init failed (%v) — channel not verified", err)
|
||
return
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cancel()
|
||
resp, err := client.Storage(ctx)
|
||
if err != nil {
|
||
logger.Printf("[WARN] local-api: GET /storage failed (%v) — channel not verified", err)
|
||
return
|
||
}
|
||
logger.Printf("[INFO] local-api: channel up (agent %s) — guest %d, %d mount(s) visible",
|
||
cfg.LocalAPI.Endpoint, resp.VMID, len(resp.Mounts))
|
||
for _, m := range resp.Mounts {
|
||
logger.Printf("[INFO] local-api: mount %s → %s (storage=%s, class=%s, backup=%v)",
|
||
m.Key, m.MountPoint, m.Storage, m.Class, m.Backup)
|
||
}
|
||
}
|
||
|
||
// runSetupMode starts the setup wizard on dual listeners and blocks until signal.
|
||
func runSetupMode(cfg *config.Config, logger *log.Logger) {
|
||
ips := setup.DetectLocalIPs()
|
||
setup.LogSetupMode(cfg.Customer.Domain, ips, cfg.Web.SetupListen, logger)
|
||
|
||
setupSrv := setup.NewServer(cfg, cfg.Paths.DataDir, logger, Version)
|
||
handler := setupSrv.Handler()
|
||
|
||
// Health endpoint wrapper (returns setup_mode: true)
|
||
healthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||
"ok": true, "message": "felhom-controller is healthy",
|
||
"setup_mode": true, "version": Version,
|
||
})
|
||
})
|
||
|
||
// Mux for both listeners
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("/api/health", healthHandler)
|
||
mux.Handle("/", handler)
|
||
|
||
// Start main listener (:8080, behind Traefik for domain access)
|
||
mainServer := &http.Server{
|
||
Addr: cfg.Web.Listen,
|
||
Handler: mux,
|
||
ReadTimeout: 30 * time.Second,
|
||
WriteTimeout: 60 * time.Second,
|
||
IdleTimeout: 120 * time.Second,
|
||
}
|
||
go func() {
|
||
logger.Printf("[INFO] Setup wizard (main) listening on %s", cfg.Web.Listen)
|
||
if err := mainServer.ListenAndServe(); err != http.ErrServerClosed {
|
||
logger.Printf("[ERROR] Main HTTP server error: %v", err)
|
||
}
|
||
}()
|
||
|
||
// Start setup-only listener (:8081, direct HTTP for LAN access)
|
||
setupServer := &http.Server{
|
||
Addr: cfg.Web.SetupListen,
|
||
Handler: mux,
|
||
ReadTimeout: 30 * time.Second,
|
||
WriteTimeout: 60 * time.Second,
|
||
IdleTimeout: 120 * time.Second,
|
||
}
|
||
go func() {
|
||
logger.Printf("[INFO] Setup wizard (LAN) listening on %s", cfg.Web.SetupListen)
|
||
if err := setupServer.ListenAndServe(); err != http.ErrServerClosed {
|
||
logger.Printf("[ERROR] Setup HTTP server error: %v", err)
|
||
}
|
||
}()
|
||
|
||
// Wait for signal
|
||
sigCh := make(chan os.Signal, 1)
|
||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||
sig := <-sigCh
|
||
logger.Printf("[INFO] Received signal %v, shutting down setup wizard...", sig)
|
||
|
||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer shutdownCancel()
|
||
mainServer.Shutdown(shutdownCtx)
|
||
setupServer.Shutdown(shutdownCtx)
|
||
logger.Println("[INFO] Setup wizard stopped")
|
||
}
|
||
|
||
// discoverHDDPaths scans deployed apps' app.yaml for HDD_PATH env values.
|
||
func discoverHDDPaths(stacksDir string, logger *log.Logger) []string {
|
||
entries, err := os.ReadDir(stacksDir)
|
||
if err != nil {
|
||
logger.Printf("[WARN] Cannot read stacks dir for HDD path discovery: %v", err)
|
||
return nil
|
||
}
|
||
seen := make(map[string]bool)
|
||
var paths []string
|
||
for _, e := range entries {
|
||
if !e.IsDir() {
|
||
continue
|
||
}
|
||
appCfg := stacks.LoadAppConfig(filepath.Join(stacksDir, e.Name()))
|
||
if appCfg == nil || !appCfg.Deployed {
|
||
continue
|
||
}
|
||
if hddPath, ok := appCfg.Env["HDD_PATH"]; ok && hddPath != "" {
|
||
cleaned := filepath.Clean(hddPath)
|
||
if !seen[cleaned] {
|
||
seen[cleaned] = true
|
||
paths = append(paths, cleaned)
|
||
}
|
||
}
|
||
}
|
||
return paths
|
||
}
|