diff --git a/CLAUDE.md b/CLAUDE.md index 852c538..24785a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,9 @@ deploy-felhom-compose/ │ ├── internal/ │ │ ├── config/ # YAML config loading │ │ ├── stacks/ # Docker Compose operations, deploy flow +│ │ ├── sync/ # Git sync — periodic pull of app catalog repo │ │ ├── api/ # REST API endpoints +│ │ ├── system/ # System info (memory, disk) │ │ └── web/ # Dashboard UI (embedded HTML/CSS templates) │ ├── Dockerfile │ ├── Makefile @@ -98,6 +100,19 @@ ssh kisfenyo@192.168.0.162 "cd /opt/docker/felhom-controller && docker pull gite - `app.yaml` persists deploy config; `deployed: true` flag controls UI state - Password fields require explicit user input or generation (no silent auto-fill) +## Git sync module (internal/sync) + +- Uses `os/exec` to call `git` CLI — no Go git library dependency +- On startup: clones repo to `{data_dir}/catalog-cache/` (shallow clone, `--depth 1`) +- Periodically: `git fetch --depth 1` + `git reset --hard origin/{branch}` +- Copies only `docker-compose.yml` and `.felhom.yml` to stacks dir +- **Never overwrites** `app.yaml` or `.env` — these contain deployed secrets +- Content-hash comparison (SHA-256) — only writes if file actually changed +- After sync, triggers `ScanStacks()` rescan for dashboard update +- `POST /api/sync` triggers immediate sync (30s debounce) +- "Sablonok frissítése" button on Alkalmazások page +- Sync status exposed in `/api/system/info` response + ## Important lessons learned 1. `PAPERLESS_OCR_LANGUAGES` (plural, with S) **installs** tesseract packs; `PAPERLESS_OCR_LANGUAGE` (singular) **selects** which to use diff --git a/CONTEXT.md b/CONTEXT.md index 7135953..fb209fa 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -7,7 +7,7 @@ > > Ask Claude Code: "Please update CONTEXT.md with what we did today" -Last updated: 2026-02-15 +Last updated: 2026-02-15 (session 2) --- @@ -28,7 +28,34 @@ Last updated: 2026-02-15 - **Running on:** demo-felhom (N100 mini PC) at 192.168.0.162:8080 - **All Phase 1 features working:** deploy, start/stop/restart/update, logs, health-aware states, auth -### What was just completed (2026-02-15) +### What was just completed (2026-02-15 session 2) +- **Phase 4: Git Sync + App Catalog Audit** — major milestone +- **Git sync module** (`internal/sync/sync.go`): + - Clones/pulls app-catalog-felhom.eu repo to local cache on startup + - Periodic sync based on `git.sync_interval` (default 15m) + - Copies `docker-compose.yml` + `.felhom.yml` to stacks dir (never overwrites `app.yaml`/`.env`) + - SHA-256 content comparison — only writes changed files + - Triggers `ScanStacks()` after sync so dashboard updates immediately + - Uses `os/exec` git CLI — no Go git library dependency +- **Manual sync button** ("Sablonok frissítése") on Alkalmazások page: + - `POST /api/sync` endpoint with 30s debounce + - Toast notification shows result (success/failure/what changed) + - Auto-reloads page if new apps or updates detected +- **Sync status** added to `/api/system/info` (last_sync, last_status, syncing flag) +- **.felhom.yml files created for all 10 apps** (paperless-ngx already had one): + - actualbudget, docmost, filebrowser, homebox, immich, mealie, romm, stirling-pdf, vaultwarden + - All follow the same format: display_name, description, category, subdomain, resources, deploy_fields +- **Docker Compose templates audited and fixed** for all 10 apps: + - Fixed `{{DOMAIN}}` → `${DOMAIN}` syntax in homebox, mealie, romm, stirling-pdf + - Fixed `{{HDD_PATH}}` → `${HDD_PATH}` in romm + - Added `deploy.resources.limits.memory` to all services across all templates + - Added `TZ=Europe/Budapest` to all sidecar services (postgres, redis, mariadb) + - Added healthcheck to romm main service + - Added `romm-redis` `condition: service_healthy` (was `service_started`) + - Standardized header comment blocks across all templates +- **Documentation updated**: app-catalog README, CLAUDE.md, CONTEXT.md + +### Previously completed (2026-02-15 session 1) - **Memory validation during deployment**: - Pre-deploy memory check: compares `mem_request` sum against usable system RAM - Hard block if requests exceed usable memory (total - 384MB reserved) @@ -66,12 +93,13 @@ Last updated: 2026-02-15 7. Documentation: restart vs up -d for image updates ### What's next (priorities) -1. Deploy a second app (e.g., Immich, Jellyfin) to validate the template system -2. Test on Raspberry Pi (pi-customer-1) -3. Add `paths.hdd_path` to demo-felhom controller.yaml to enable HDD bar -4. Add memory limits + `mem_request`/`mem_limit` to other app catalog templates (Immich, Jellyfin, etc.) -5. Phase 2 continued: CPU/temperature metrics, Healthchecks.io pings -6. Phase 3: Backup system (DB dumps + restic) +1. Build + deploy the updated controller with git sync module +2. Deploy a second app (e.g., ActualBudget — simplest, or Immich — tests HDD + secrets) to validate all .felhom.yml files +3. Test git sync end-to-end: push a template change to app-catalog, verify controller picks it up +4. Test on Raspberry Pi (pi-customer-1) +5. Add `paths.hdd_path` to demo-felhom controller.yaml to enable HDD bar +6. Phase 2 continued: CPU/temperature metrics, Healthchecks.io pings +7. Phase 3: Backup system (DB dumps + restic) ## Architecture decisions @@ -90,6 +118,9 @@ Last updated: 2026-02-15 | mem_request vs mem_limit (K8s-inspired) | Requests = expected usage (hard block), limits = peak (overcommit OK) | | 384MB reserved for system | Prevents deploying apps that would starve the OS/controller | | Logo SVG embedded as Go constant | Same approach as CSS/HTML — zero external file deps | +| Git sync via os/exec git CLI | No Go git library needed, git is in the container image | +| SHA-256 for content comparison | Only copy changed files, avoid unnecessary disk writes | +| 30s debounce on manual sync | Prevents spamming the git server | ## Key file locations on demo-felhom @@ -120,7 +151,7 @@ Last updated: 2026-02-15 | Repository | Status | Notes | |------------|--------|-------| | deploy-felhom-compose | Active | This repo. Controller code + deploy scripts | -| app-catalog-felhom.eu | Active | 49 app templates, paperless-ngx has memory limits | +| app-catalog-felhom.eu | Active | 10 app templates, all with .felhom.yml metadata + memory limits | | felhom.eu | Stable | Website live, SEO indexed, email working | | homelab-manifests | Stable | k3s cluster running (dooplex.hu services) | | misc-scripts | Utility | collect-repo.sh, backup helpers | diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 1646ff6..da4e084 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -14,6 +14,7 @@ import ( "gitea.dooplex.hu/admin/felhom-controller/internal/api" "gitea.dooplex.hu/admin/felhom-controller/internal/config" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" + catalogsync "gitea.dooplex.hu/admin/felhom-controller/internal/sync" "gitea.dooplex.hu/admin/felhom-controller/internal/web" ) @@ -55,8 +56,13 @@ func main() { logger.Printf("[WARN] Initial stack scan failed: %v", err) } + // --- Initialize catalog syncer --- + syncer := catalogsync.New(cfg, logger, stackMgr.ScanStacks) + syncer.Start() + defer syncer.Stop() + // --- Initialize API router --- - apiRouter := api.NewRouter(cfg, stackMgr, logger) + apiRouter := api.NewRouter(cfg, stackMgr, syncer, logger) // --- Initialize web server --- webServer := web.NewServer(cfg, stackMgr, logger, Version) diff --git a/controller/internal/api/router.go b/controller/internal/api/router.go index 3bf4615..66443dc 100644 --- a/controller/internal/api/router.go +++ b/controller/internal/api/router.go @@ -10,6 +10,7 @@ import ( "gitea.dooplex.hu/admin/felhom-controller/internal/config" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" + catalogsync "gitea.dooplex.hu/admin/felhom-controller/internal/sync" "gitea.dooplex.hu/admin/felhom-controller/internal/system" ) @@ -17,11 +18,12 @@ import ( type Router struct { cfg *config.Config stackMgr *stacks.Manager + syncer *catalogsync.Syncer logger *log.Logger } -func NewRouter(cfg *config.Config, stackMgr *stacks.Manager, logger *log.Logger) *Router { - return &Router{cfg: cfg, stackMgr: stackMgr, logger: logger} +func NewRouter(cfg *config.Config, stackMgr *stacks.Manager, syncer *catalogsync.Syncer, logger *log.Logger) *Router { + return &Router{cfg: cfg, stackMgr: stackMgr, syncer: syncer, logger: logger} } type apiResponse struct { @@ -77,6 +79,10 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { case hasSuffix(path, "/logs") && req.Method == http.MethodGet: r.getStackLogs(w, req, extractName(path, "/logs")) + // POST /api/sync — trigger immediate catalog sync + case path == "/sync" && req.Method == http.MethodPost: + r.triggerSync(w, req) + // GET /api/system/info case path == "/system/info" && req.Method == http.MethodGet: r.systemInfo(w, req) @@ -220,9 +226,25 @@ func (r *Router) getStackLogs(w http.ResponseWriter, req *http.Request, name str writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: map[string]string{"logs": output}}) } +func (r *Router) triggerSync(w http.ResponseWriter, _ *http.Request) { + r.logger.Println("[API] Manual catalog sync requested") + result := r.syncer.TriggerSync() + if !result.OK { + writeJSON(w, http.StatusTooManyRequests, apiResponse{OK: false, Error: result.Message}) + return + } + r.logger.Printf("[API] Catalog sync completed: %s", result.Message) + writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: result.Message, Data: result}) +} + func (r *Router) systemInfo(w http.ResponseWriter, _ *http.Request) { info := system.GetInfo(r.cfg.Paths.HDDPath) - writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: info}) + syncStatus := r.syncer.Status() + data := map[string]interface{}{ + "system": info, + "sync_status": syncStatus, + } + writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: data}) } // --- Helpers --- diff --git a/controller/internal/sync/sync.go b/controller/internal/sync/sync.go new file mode 100644 index 0000000..5842dbe --- /dev/null +++ b/controller/internal/sync/sync.go @@ -0,0 +1,355 @@ +package sync + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/config" +) + +// Syncer handles periodic git sync of the app catalog to the local stacks directory. +type Syncer struct { + cfg *config.Config + logger *log.Logger + cacheDir string // local git clone + rescanFn func() error + mu sync.Mutex + lastSync time.Time + lastErr error + syncing bool + stopCh chan struct{} +} + +// SyncStatus holds information about the last sync operation. +type SyncStatus struct { + LastSync time.Time `json:"last_sync"` + LastStatus string `json:"last_status"` // "ok", "error", "disabled", "never" + LastError string `json:"last_error,omitempty"` + Syncing bool `json:"syncing"` +} + +// SyncResult holds the result of a single sync operation. +type SyncResult struct { + OK bool `json:"ok"` + NewApps []string `json:"new_apps,omitempty"` + Updated []string `json:"updated,omitempty"` + Message string `json:"message"` +} + +// New creates a new Syncer. rescanFn is called after a successful sync to trigger ScanStacks(). +func New(cfg *config.Config, logger *log.Logger, rescanFn func() error) *Syncer { + cacheDir := filepath.Join(cfg.Paths.DataDir, "catalog-cache") + return &Syncer{ + cfg: cfg, + logger: logger, + cacheDir: cacheDir, + rescanFn: rescanFn, + stopCh: make(chan struct{}), + } +} + +// Start begins the periodic sync loop. Call Stop() to terminate. +func (s *Syncer) Start() { + if s.cfg.Git.RepoURL == "" { + s.logger.Println("[SYNC] Git repo URL is empty — sync disabled (manual mode)") + return + } + + interval, err := time.ParseDuration(s.cfg.Git.SyncInterval) + if err != nil { + s.logger.Printf("[SYNC] Invalid sync_interval %q, defaulting to 15m", s.cfg.Git.SyncInterval) + interval = 15 * time.Minute + } + + s.logger.Printf("[SYNC] Starting catalog sync (repo: %s, interval: %s)", s.cfg.Git.RepoURL, interval) + + // Initial sync on startup + go func() { + result := s.doSync() + s.logger.Printf("[SYNC] Initial sync: %s", result.Message) + }() + + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-s.stopCh: + s.logger.Println("[SYNC] Sync loop stopped") + return + case <-ticker.C: + result := s.doSync() + s.logger.Printf("[SYNC] Periodic sync: %s", result.Message) + } + } + }() +} + +// Stop terminates the periodic sync loop. +func (s *Syncer) Stop() { + close(s.stopCh) +} + +// TriggerSync performs an immediate sync. Returns the result. +// Debounce: refuses if last sync was less than 30 seconds ago. +func (s *Syncer) TriggerSync() SyncResult { + if s.cfg.Git.RepoURL == "" { + return SyncResult{OK: false, Message: "Git sync is disabled (no repo_url configured)"} + } + + s.mu.Lock() + if s.syncing { + s.mu.Unlock() + return SyncResult{OK: false, Message: "Szinkronizálás már folyamatban"} + } + if time.Since(s.lastSync) < 30*time.Second { + s.mu.Unlock() + return SyncResult{OK: false, Message: "Túl gyakori szinkronizálás — várj 30 másodpercet"} + } + s.mu.Unlock() + + return s.doSync() +} + +// Status returns the current sync status. +func (s *Syncer) Status() SyncStatus { + s.mu.Lock() + defer s.mu.Unlock() + + status := SyncStatus{ + LastSync: s.lastSync, + Syncing: s.syncing, + } + + if s.cfg.Git.RepoURL == "" { + status.LastStatus = "disabled" + } else if s.lastSync.IsZero() { + status.LastStatus = "never" + } else if s.lastErr != nil { + status.LastStatus = "error" + status.LastError = s.lastErr.Error() + } else { + status.LastStatus = "ok" + } + + return status +} + +// doSync performs the actual git clone/pull + file copy. +func (s *Syncer) doSync() SyncResult { + s.mu.Lock() + s.syncing = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.syncing = false + s.mu.Unlock() + }() + + result := SyncResult{OK: true} + + // Step 1: Clone or pull + if err := s.gitCloneOrPull(); err != nil { + s.mu.Lock() + s.lastErr = err + s.lastSync = time.Now() + s.mu.Unlock() + return SyncResult{OK: false, Message: fmt.Sprintf("Git hiba: %v", err)} + } + + // Step 2: Copy templates to stacks dir + newApps, updated, err := s.copyTemplates() + if err != nil { + s.mu.Lock() + s.lastErr = err + s.lastSync = time.Now() + s.mu.Unlock() + return SyncResult{OK: false, Message: fmt.Sprintf("Másolási hiba: %v", err)} + } + + result.NewApps = newApps + result.Updated = updated + + // Step 3: Trigger rescan if anything changed + if len(newApps) > 0 || len(updated) > 0 { + if err := s.rescanFn(); err != nil { + s.logger.Printf("[SYNC] Rescan after sync failed: %v", err) + } + } + + // Build message + parts := []string{} + if len(newApps) > 0 { + parts = append(parts, fmt.Sprintf("új: %s", strings.Join(newApps, ", "))) + } + if len(updated) > 0 { + parts = append(parts, fmt.Sprintf("frissítve: %s", strings.Join(updated, ", "))) + } + if len(parts) == 0 { + result.Message = "Sablonok naprakészek — nincs változás" + } else { + result.Message = "Sablonok frissítve — " + strings.Join(parts, "; ") + } + + s.mu.Lock() + s.lastErr = nil + s.lastSync = time.Now() + s.mu.Unlock() + + return result +} + +// gitCloneOrPull clones the repo if not yet cloned, or pulls latest changes. +func (s *Syncer) gitCloneOrPull() error { + if err := os.MkdirAll(filepath.Dir(s.cacheDir), 0755); err != nil { + return fmt.Errorf("creating cache parent dir: %w", err) + } + + gitDir := filepath.Join(s.cacheDir, ".git") + if _, err := os.Stat(gitDir); os.IsNotExist(err) { + // Clone + s.logger.Printf("[SYNC] Cloning %s → %s", s.cfg.Git.RepoURL, s.cacheDir) + args := []string{"clone", "--depth", "1", "--branch", s.cfg.Git.Branch} + repoURL := s.buildRepoURL() + args = append(args, repoURL, s.cacheDir) + return s.runGit(args...) + } + + // Pull + s.logger.Printf("[SYNC] Pulling latest from %s (branch: %s)", s.cfg.Git.RepoURL, s.cfg.Git.Branch) + if err := s.runGitInDir(s.cacheDir, "fetch", "--depth", "1", "origin", s.cfg.Git.Branch); err != nil { + return fmt.Errorf("git fetch: %w", err) + } + if err := s.runGitInDir(s.cacheDir, "reset", "--hard", "origin/"+s.cfg.Git.Branch); err != nil { + return fmt.Errorf("git reset: %w", err) + } + return nil +} + +// buildRepoURL constructs the repo URL with optional auth credentials. +func (s *Syncer) buildRepoURL() string { + url := s.cfg.Git.RepoURL + if s.cfg.Git.Username != "" && s.cfg.Git.Token != "" { + // Inject credentials into HTTPS URL: https://user:token@host/path + url = strings.Replace(url, "https://", fmt.Sprintf("https://%s:%s@", s.cfg.Git.Username, s.cfg.Git.Token), 1) + } + return url +} + +// copyTemplates copies docker-compose.yml and .felhom.yml from the catalog cache +// to the stacks directory. Never overwrites app.yaml or .env files. +func (s *Syncer) copyTemplates() (newApps []string, updated []string, err error) { + templatesDir := filepath.Join(s.cacheDir, "templates") + entries, err := os.ReadDir(templatesDir) + if err != nil { + return nil, nil, fmt.Errorf("reading templates dir: %w", err) + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + appName := entry.Name() + srcDir := filepath.Join(templatesDir, appName) + dstDir := filepath.Join(s.cfg.Paths.StacksDir, appName) + + isNew := false + if _, err := os.Stat(dstDir); os.IsNotExist(err) { + isNew = true + if err := os.MkdirAll(dstDir, 0755); err != nil { + s.logger.Printf("[SYNC] Failed to create stack dir %s: %v", dstDir, err) + continue + } + } + + // Files to sync (only template files, never app.yaml or .env) + syncFiles := []string{"docker-compose.yml", ".felhom.yml"} + anyChanged := false + + for _, filename := range syncFiles { + src := filepath.Join(srcDir, filename) + dst := filepath.Join(dstDir, filename) + + if _, err := os.Stat(src); os.IsNotExist(err) { + continue + } + + changed, err := copyIfChanged(src, dst) + if err != nil { + s.logger.Printf("[SYNC] Failed to copy %s/%s: %v", appName, filename, err) + continue + } + if changed { + anyChanged = true + s.logger.Printf("[SYNC] Updated %s/%s", appName, filename) + } + } + + if isNew { + newApps = append(newApps, appName) + } else if anyChanged { + updated = append(updated, appName) + } + } + + return newApps, updated, nil +} + +// copyIfChanged copies src to dst only if the content differs. +// Returns true if the file was actually written. +func copyIfChanged(src, dst string) (bool, error) { + srcData, err := os.ReadFile(src) + if err != nil { + return false, fmt.Errorf("reading %s: %w", src, err) + } + + // Check if dst exists and has same content + dstData, err := os.ReadFile(dst) + if err == nil { + srcHash := sha256.Sum256(srcData) + dstHash := sha256.Sum256(dstData) + if srcHash == dstHash { + return false, nil // No change + } + } + + if err := os.WriteFile(dst, srcData, 0644); err != nil { + return false, fmt.Errorf("writing %s: %w", dst, err) + } + return true, nil +} + +// runGit executes a git command with the given args. +func (s *Syncer) runGit(args ...string) error { + return s.runGitInDir("", args...) +} + +// runGitInDir executes a git command in the specified directory. +func (s *Syncer) runGitInDir(dir string, args ...string) error { + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + + s.logger.Printf("[SYNC] Running: git %s", strings.Join(args, " ")) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("git %s: %w\nstderr: %s", strings.Join(args, " "), err, stderr.String()) + } + return nil +} diff --git a/controller/internal/web/templates.go b/controller/internal/web/templates.go index 2c737d5..97ab7b1 100644 --- a/controller/internal/web/templates.go +++ b/controller/internal/web/templates.go @@ -37,6 +37,41 @@ const layoutTmpl = ` {{define "layout_end"}}