From 44a7d0de2c30b685760fe2f3c4fedff8552811ec Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sat, 14 Feb 2026 12:41:08 +0100 Subject: [PATCH] updated memory calculation and logo --- CONTEXT.md | 26 ++++- controller/README.md | 30 +++++ controller/internal/api/router.go | 11 +- controller/internal/config/config.go | 6 + controller/internal/stacks/deploy.go | 79 +++++++++---- controller/internal/stacks/manager.go | 64 +++++++++++ controller/internal/system/info_linux.go | 11 ++ controller/internal/system/info_other.go | 7 ++ controller/internal/web/server.go | 38 +++++- controller/internal/web/templates.go | 140 ++++++++++++++++++++--- 10 files changed, 365 insertions(+), 47 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index f7a78c6..7135953 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -7,7 +7,7 @@ > > Ask Claude Code: "Please update CONTEXT.md with what we did today" -Last updated: 2026-02-14 (evening) +Last updated: 2026-02-15 --- @@ -28,7 +28,20 @@ Last updated: 2026-02-14 (evening) - **Running on:** demo-felhom (N100 mini PC) at 192.168.0.162:8080 - **All Phase 1 features working:** deploy, start/stop/restart/update, logs, health-aware states, auth -### What was just completed (2026-02-14) +### What was just completed (2026-02-15) +- **Memory validation during deployment**: + - Pre-deploy memory check: compares `mem_request` sum against usable system RAM + - Hard block if requests exceed usable memory (total - 384MB reserved) + - Soft warning if `mem_limit` sum exceeds total RAM (overcommit OK for limits) + - `ParseMemoryMB()` supports "500M", "1G", "1.5G", "1024" formats + - `CommittedMemory()` sums requests/limits across all deployed stacks + - Memory summary bar shown on deploy page before user clicks deploy + - `system.reserved_memory_mb` configurable in controller.yaml (default: 384) +- **Display: `~` prefix on mem_request** in UI badges (display-only, exact value stored) +- **Felhom.eu logo** replaced text logos in sidebar and login page with actual SVG logo + - Logo SVG embedded as Go string constant, served at `/static/felhom-logo.svg` + +### Previously completed (2026-02-14) - **System info bar on Vezérlőpult dashboard**: RAM, SSD, and optional HDD usage - Progress bars with color coding (green < 70%, yellow 70-85%, red > 85%) - New `internal/system` package reads `/proc/meminfo` + `syscall.Statfs` @@ -56,9 +69,9 @@ Last updated: 2026-02-14 (evening) 1. Deploy a second app (e.g., Immich, Jellyfin) to validate the template system 2. Test on Raspberry Pi (pi-customer-1) 3. Add `paths.hdd_path` to demo-felhom controller.yaml to enable HDD bar -4. Phase 2 continued: CPU/temperature metrics, Healthchecks.io pings -5. Phase 3: Backup system (DB dumps + restic) -6. Add memory limits to other app catalog templates (Immich, Jellyfin, etc.) +4. Add memory limits + `mem_request`/`mem_limit` to other app catalog templates (Immich, Jellyfin, etc.) +5. Phase 2 continued: CPU/temperature metrics, Healthchecks.io pings +6. Phase 3: Backup system (DB dumps + restic) ## Architecture decisions @@ -74,6 +87,9 @@ Last updated: 2026-02-14 (evening) | Health-aware state from Docker Status field | Docker's State says "running" even for unhealthy containers | | Memory limits via deploy.resources.limits | Prevents runaway containers; ~50% headroom over expected usage | | System info from /proc/meminfo + statfs | No external dependencies, cheap to read on each page load | +| mem_request vs mem_limit (K8s-inspired) | Requests = expected usage (hard block), limits = peak (overcommit OK) | +| 384MB reserved for system | Prevents deploying apps that would starve the OS/controller | +| Logo SVG embedded as Go constant | Same approach as CSS/HTML — zero external file deps | ## Key file locations on demo-felhom diff --git a/controller/README.md b/controller/README.md index f5c53b0..0cce841 100644 --- a/controller/README.md +++ b/controller/README.md @@ -38,6 +38,9 @@ Current version: **v0.2.1** - Protected stacks (traefik, cloudflared, felhom-controller) can't be stopped - System info bar on dashboard: RAM, SSD, and HDD usage with progress bars - Docker Compose memory limits enforced via `deploy.resources.limits.memory` +- Pre-deploy memory validation (hard block on `mem_request` overcommit, soft warning on `mem_limit` overcommit) +- Memory summary bar shown on deploy page before deployment +- Felhom.eu logo SVG in sidebar and login page ### Known issues / next priorities - Cloudflare Tunnel + Traefik TLS: paperless.demo-felhom.eu works locally but shows "Not secure" (certificate chain not fully validated through tunnel) @@ -139,6 +142,7 @@ controller/ - **User input**: HDD path, admin password, language, etc. - **"🎲 Generálás"** button next to password fields 3. Clicks "Telepítés" → controller: + - **Memory validation**: checks `mem_request` against available system RAM (see below) - Validates all required fields (password fields must be explicitly filled or generated) - Generates auto-secrets (DB passwords, hex keys) - Saves `app.yaml` (env vars + locked fields list) @@ -147,6 +151,30 @@ controller/ 4. Post-deploy: locked fields (DB_PASSWORD, etc.) become read-only 5. "Részletek" button opens deploy page in read-only mode showing current config +### Memory validation during deploy + +Before deploying an app, the controller checks if there's enough RAM. This uses the Kubernetes-inspired `mem_request` / `mem_limit` model: + +| Field | In `.felhom.yml` | Purpose | Validation | +|-------|-------------------|---------|------------| +| `mem_request` | `resources.mem_request: "500M"` | Expected memory usage during normal operation | **Hard block** — sum of requests must not exceed usable RAM | +| `mem_limit` | `resources.mem_limit: "1152M"` | Docker `deploy.resources.limits.memory` total across all containers | **Soft warning** — overcommit is allowed for limits | +| `pi_compatible` | `resources.pi_compatible: true` | Whether the app can run on Raspberry Pi | Display-only hint | +| `needs_hdd` | `resources.needs_hdd: true` | Whether the app needs external storage | Display-only hint | + +**How it works:** +- `usable_memory = total_ram - reserved_memory_mb` (default: 384MB reserved for OS + controller) +- If `sum(deployed mem_requests) + new_mem_request > usable_memory` → deploy is **blocked** with error +- If `sum(deployed mem_limits) + new_mem_limit > total_ram` → deploy proceeds with **warning** +- Apps without `mem_request` set are treated as 0MB (never blocked) +- The deploy page shows a memory summary bar before the user clicks deploy + +Configure the reserved memory via `controller.yaml`: +```yaml +system: + reserved_memory_mb: 384 # default +``` + ### Container state display The dashboard shows health-aware container states with distinct colors: @@ -314,6 +342,8 @@ docker compose up -d ### Phase 2 — Monitoring & Health - [x] System metrics on dashboard (RAM, SSD, HDD usage bars) - [x] `/api/system/info` endpoint with live resource data +- [x] Pre-deploy memory validation (mem_request hard block, mem_limit soft warning) +- [x] Memory summary bar on deploy page - [ ] CPU and temperature metrics - [ ] Healthchecks.io ping integration - [ ] Customer notifications (email/Telegram) diff --git a/controller/internal/api/router.go b/controller/internal/api/router.go index 6595f9c..3bf4615 100644 --- a/controller/internal/api/router.go +++ b/controller/internal/api/router.go @@ -153,20 +153,25 @@ func (r *Router) deployStack(w http.ResponseWriter, req *http.Request, name stri Values: body.Values, } - if err := r.stackMgr.DeployStack(deployReq); err != nil { + warning, err := r.stackMgr.DeployStack(deployReq) + if err != nil { r.logger.Printf("[API] Deploy failed for %s: %v", name, err) status := http.StatusInternalServerError if strings.Contains(err.Error(), "already deployed") { status = http.StatusConflict } - if strings.Contains(err.Error(), "required field") || strings.Contains(err.Error(), "does not exist") || strings.Contains(err.Error(), "kötelező") { + if strings.Contains(err.Error(), "required field") || strings.Contains(err.Error(), "does not exist") || strings.Contains(err.Error(), "kötelező") || strings.Contains(err.Error(), "memória") { status = http.StatusBadRequest } writeJSON(w, status, apiResponse{OK: false, Error: err.Error()}) return } - writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: "Stack " + name + " deployed"}) + resp := apiResponse{OK: true, Message: "Stack " + name + " deployed"} + if warning != "" { + resp.Data = map[string]string{"warning": warning} + } + writeJSON(w, http.StatusOK, resp) } func (r *Router) actionStack(w http.ResponseWriter, action, name string) { diff --git a/controller/internal/config/config.go b/controller/internal/config/config.go index 429db99..f395176 100644 --- a/controller/internal/config/config.go +++ b/controller/internal/config/config.go @@ -24,6 +24,11 @@ type Config struct { Notifications NotificationsConfig `yaml:"notifications"` Logging LoggingConfig `yaml:"logging"` Assets AssetsConfig `yaml:"assets"` + System SystemConfig `yaml:"system"` +} + +type SystemConfig struct { + ReservedMemoryMB int `yaml:"reserved_memory_mb"` } type CustomerConfig struct { @@ -196,6 +201,7 @@ func applyDefaults(cfg *Config) { di(&cfg.Logging.MaxSizeMB, 10) di(&cfg.Logging.MaxFiles, 3) d(&cfg.Assets.SourceURL, "https://felhom.eu") + di(&cfg.System.ReservedMemoryMB, 384) } func applyEnvOverrides(cfg *Config) { diff --git a/controller/internal/stacks/deploy.go b/controller/internal/stacks/deploy.go index f828315..da76314 100644 --- a/controller/internal/stacks/deploy.go +++ b/controller/internal/stacks/deploy.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "gitea.dooplex.hu/admin/felhom-controller/internal/system" "gopkg.in/yaml.v3" ) @@ -28,18 +29,20 @@ type DeployRequest struct { Values map[string]string `json:"values"` // env_var -> user-provided value } -// DeployStack handles first-time deployment of an app: -// 1. Load metadata (.felhom.yml) to know what fields exist -// 2. Auto-generate secrets for secret fields (hidden from user) -// 3. Auto-fill domain from controller config -// 4. Validate all user-provided values (password, path, required fields) -// 5. Save app.yaml -// 6. Run docker compose up -d with env vars -// 7. Update in-memory stack state -func (m *Manager) DeployStack(req DeployRequest) error { +// DeployStack handles first-time deployment of an app. +// Returns a warning message (empty if none) and an error if deployment is blocked. +// 1. Check available memory against app requirements +// 2. Load metadata (.felhom.yml) to know what fields exist +// 3. Auto-generate secrets for secret fields (hidden from user) +// 4. Auto-fill domain from controller config +// 5. Validate all user-provided values (password, path, required fields) +// 6. Save app.yaml +// 7. Run docker compose up -d with env vars +// 8. Update in-memory stack state +func (m *Manager) DeployStack(req DeployRequest) (string, error) { stack, ok := m.GetStack(req.StackName) if !ok { - return fmt.Errorf("stack %q not found", req.StackName) + return "", fmt.Errorf("stack %q not found", req.StackName) } stackDir := filepath.Dir(stack.ComposePath) @@ -48,7 +51,43 @@ func (m *Manager) DeployStack(req DeployRequest) error { // Check if already deployed existing := LoadAppConfig(stackDir) if existing != nil && existing.Deployed { - return fmt.Errorf("stack %q is already deployed; use update instead", req.StackName) + return "", fmt.Errorf("stack %q is already deployed; use update instead", req.StackName) + } + + // --- Memory validation --- + var deployWarning string + reservedMB := m.cfg.System.ReservedMemoryMB + totalMB, memErr := system.GetTotalMemoryMB() + if memErr != nil { + m.logger.Printf("[WARN] Cannot read system memory: %v — skipping memory check", memErr) + } else { + usableMB := totalMB - reservedMB + currentReqMB, currentLimitMB := m.CommittedMemory() + newReqMB := ParseMemoryMB(meta.Resources.MemRequest) + newLimitMB := ParseMemoryMB(meta.Resources.MemLimit) + + m.logger.Printf("[INFO] Memory check: total=%dMB, reserved=%dMB, usable=%dMB, committed_req=%dMB, new_req=%dMB, remaining=%dMB", + totalMB, reservedMB, usableMB, currentReqMB, newReqMB, usableMB-currentReqMB-newReqMB) + + // Hard block: requests exceed usable memory + if newReqMB > 0 && currentReqMB+newReqMB > usableMB { + return "", fmt.Errorf( + "Nincs elég memória az alkalmazás telepítéséhez. "+ + "Szükséges: %d MB, Elérhető: %d MB "+ + "(összesen: %d MB, ebből %d MB már foglalt, %d MB rendszer számára fenntartva)", + newReqMB, + usableMB-currentReqMB, + totalMB, + currentReqMB, + reservedMB, + ) + } + + // Soft warning: limits exceed total (overcommit) + if newLimitMB > 0 && currentLimitMB+newLimitMB > totalMB { + deployWarning = "Az alkalmazások csúcsterhelése meghaladhatja a rendelkezésre álló memóriát. " + + "Normál használat mellett ez nem okoz problémát." + } } // Debug: log received values (redact passwords/secrets) @@ -77,7 +116,7 @@ func (m *Manager) DeployStack(req DeployRequest) error { // Always auto-generate, user never sees these generated, err := generateValue(field.Generate) if err != nil { - return fmt.Errorf("generating %s: %w", field.EnvVar, err) + return "", fmt.Errorf("generating %s: %w", field.EnvVar, err) } value = generated @@ -87,7 +126,7 @@ func (m *Manager) DeployStack(req DeployRequest) error { if userVal, ok := req.Values[field.EnvVar]; ok && userVal != "" { value = userVal } else { - return fmt.Errorf("a(z) %q mező kitöltése kötelező — használja a Generálás gombot vagy írjon be egy jelszót", field.Label) + return "", fmt.Errorf("a(z) %q mező kitöltése kötelező — használja a Generálás gombot vagy írjon be egy jelszót", field.Label) } default: @@ -101,13 +140,13 @@ func (m *Manager) DeployStack(req DeployRequest) error { // Validate required fields if field.Required && value == "" { - return fmt.Errorf("a(z) %q (%s) mező kitöltése kötelező", field.Label, field.EnvVar) + return "", fmt.Errorf("a(z) %q (%s) mező kitöltése kötelező", field.Label, field.EnvVar) } // Validate path fields exist on disk (inside the container's filesystem) if field.Type == "path" && value != "" { if _, err := os.Stat(value); os.IsNotExist(err) { - return fmt.Errorf("path %q does not exist for field %q", value, field.Label) + return "", fmt.Errorf("path %q does not exist for field %q", value, field.Label) } } @@ -129,7 +168,7 @@ func (m *Manager) DeployStack(req DeployRequest) error { } if err := SaveAppConfig(stackDir, appCfg); err != nil { - return fmt.Errorf("saving app config: %w", err) + return "", fmt.Errorf("saving app config: %w", err) } // Debug: log final env var keys (not values) @@ -140,12 +179,12 @@ func (m *Manager) DeployStack(req DeployRequest) error { m.logger.Printf("[INFO] Deploying stack %s with %d env vars: [%s]", req.StackName, len(env), strings.Join(envKeys, ", ")) // Run docker compose up -d - _, err := m.composeExecWithEnv(stackDir, env, "up", "-d") - if err != nil { + _, composeErr := m.composeExecWithEnv(stackDir, env, "up", "-d") + if composeErr != nil { // Deployment failed — keep app.yaml for debugging but mark as not deployed appCfg.Deployed = false _ = SaveAppConfig(stackDir, appCfg) - return fmt.Errorf("docker compose up failed: %w", err) + return "", fmt.Errorf("docker compose up failed: %w", composeErr) } // Update in-memory stack state immediately so the UI reflects the deployment @@ -158,7 +197,7 @@ func (m *Manager) DeployStack(req DeployRequest) error { m.mu.Unlock() m.logger.Printf("[INFO] Stack %s deployed successfully", req.StackName) - return m.RefreshStatus() + return deployWarning, m.RefreshStatus() } // UpdateStackConfig updates non-locked fields for a deployed stack. diff --git a/controller/internal/stacks/manager.go b/controller/internal/stacks/manager.go index d9626be..f0a48ad 100644 --- a/controller/internal/stacks/manager.go +++ b/controller/internal/stacks/manager.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -516,4 +517,67 @@ func (m *Manager) execCommand(name string, args ...string) (string, error) { } return stdout.String(), nil +} + +// --- Memory helpers --- + +// ParseMemoryMB parses a memory string like "500M", "1G", "1.5G", "1024M", "768" +// into megabytes. Returns 0 for empty or unparseable values. Case-insensitive. +func ParseMemoryMB(s string) int { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + upper := strings.ToUpper(s) + + if strings.HasSuffix(upper, "GB") { + val, err := strconv.ParseFloat(strings.TrimSuffix(upper, "GB"), 64) + if err != nil { + return 0 + } + return int(val * 1024) + } + if strings.HasSuffix(upper, "G") { + val, err := strconv.ParseFloat(strings.TrimSuffix(upper, "G"), 64) + if err != nil { + return 0 + } + return int(val * 1024) + } + if strings.HasSuffix(upper, "MB") { + val, err := strconv.ParseFloat(strings.TrimSuffix(upper, "MB"), 64) + if err != nil { + return 0 + } + return int(val) + } + if strings.HasSuffix(upper, "M") { + val, err := strconv.ParseFloat(strings.TrimSuffix(upper, "M"), 64) + if err != nil { + return 0 + } + return int(val) + } + + // Plain number — assume MB + val, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return int(val) +} + +// CommittedMemory returns the sum of mem_request and mem_limit across all deployed stacks. +func (m *Manager) CommittedMemory() (requestMB int, limitMB int) { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, s := range m.stacks { + if !s.Deployed { + continue + } + requestMB += ParseMemoryMB(s.Meta.Resources.MemRequest) + limitMB += ParseMemoryMB(s.Meta.Resources.MemLimit) + } + return } \ No newline at end of file diff --git a/controller/internal/system/info_linux.go b/controller/internal/system/info_linux.go index 05f676f..395686b 100644 --- a/controller/internal/system/info_linux.go +++ b/controller/internal/system/info_linux.go @@ -4,6 +4,7 @@ package system import ( "bufio" + "fmt" "os" "strings" "syscall" @@ -29,6 +30,16 @@ func GetInfo(hddPath string) SystemInfo { return info } +// GetTotalMemoryMB reads total system memory from /proc/meminfo. +func GetTotalMemoryMB() (int, error) { + info := SystemInfo{} + readMemInfo(&info) + if info.TotalMemMB == 0 { + return 0, fmt.Errorf("could not read MemTotal from /proc/meminfo") + } + return int(info.TotalMemMB), nil +} + func readMemInfo(info *SystemInfo) { f, err := os.Open("/proc/meminfo") if err != nil { diff --git a/controller/internal/system/info_other.go b/controller/internal/system/info_other.go index edb7b94..42998c9 100644 --- a/controller/internal/system/info_other.go +++ b/controller/internal/system/info_other.go @@ -2,7 +2,14 @@ package system +import "fmt" + // GetInfo returns empty system info on non-Linux platforms. func GetInfo(_ string) SystemInfo { return SystemInfo{} } + +// GetTotalMemoryMB is not available on non-Linux platforms. +func GetTotalMemoryMB() (int, error) { + return 0, fmt.Errorf("/proc/meminfo not available on this platform") +} diff --git a/controller/internal/web/server.go b/controller/internal/web/server.go index fdb3c5a..92fe2f7 100644 --- a/controller/internal/web/server.go +++ b/controller/internal/web/server.go @@ -184,6 +184,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/css") w.Header().Set("Cache-Control", "public, max-age=3600") fmt.Fprint(w, cssContent) + case path == "/static/felhom-logo.svg": + w.Header().Set("Content-Type", "image/svg+xml") + w.Header().Set("Cache-Control", "public, max-age=86400") + fmt.Fprint(w, felhomLogoSVG) case strings.HasPrefix(path, "/static/assets/"): s.serveAsset(w, r, strings.TrimPrefix(path, "/static/assets/")) case strings.HasPrefix(path, "/apps/"): @@ -388,18 +392,50 @@ func (s *Server) deployHandler(w http.ResponseWriter, _ *http.Request, name stri } stack, _ := s.stackMgr.GetStack(name) + alreadyDeployed := appCfg != nil && appCfg.Deployed data := s.baseData("deploy", meta.DisplayName+" — Telepítés") data["Stack"] = stack data["Meta"] = meta data["AppConfig"] = appCfg - data["AlreadyDeployed"] = appCfg != nil && appCfg.Deployed + data["AlreadyDeployed"] = alreadyDeployed data["LogoURL"] = s.cfg.AppLogoURL(meta.Slug) data["LogoPNGURL"] = s.cfg.AppLogoPNGURL(meta.Slug) data["AppPageURL"] = s.cfg.AppPageURL(meta.Slug) data["UserFields"] = meta.UserFacingFields() data["AutoFields"] = meta.AutoGeneratedFields() + // Memory info for deploy page (only for non-deployed apps) + if !alreadyDeployed { + memInfo := map[string]interface{}{"Available": false} + totalMB, memErr := system.GetTotalMemoryMB() + if memErr == nil { + reservedMB := s.cfg.System.ReservedMemoryMB + usableMB := totalMB - reservedMB + committedReqMB, committedLimitMB := s.stackMgr.CommittedMemory() + newReqMB := stacks.ParseMemoryMB(meta.Resources.MemRequest) + newLimitMB := stacks.ParseMemoryMB(meta.Resources.MemLimit) + afterReqMB := committedReqMB + newReqMB + afterLimitMB := committedLimitMB + newLimitMB + percent := 0 + if usableMB > 0 { + percent = afterReqMB * 100 / usableMB + } + + memInfo["Available"] = true + memInfo["TotalMB"] = totalMB + memInfo["ReservedMB"] = reservedMB + memInfo["UsableMB"] = usableMB + memInfo["CommittedMB"] = committedReqMB + memInfo["NewRequestMB"] = newReqMB + memInfo["AfterMB"] = afterReqMB + memInfo["Percent"] = percent + memInfo["Blocked"] = newReqMB > 0 && afterReqMB > usableMB + memInfo["OvercommitWarn"] = newLimitMB > 0 && afterLimitMB > totalMB + } + data["MemoryInfo"] = memInfo + } + s.render(w, "deploy", data) } diff --git a/controller/internal/web/templates.go b/controller/internal/web/templates.go index e98ec85..2c737d5 100644 --- a/controller/internal/web/templates.go +++ b/controller/internal/web/templates.go @@ -19,7 +19,7 @@ const layoutTmpl = `