From 82b55c73a21d4040fad3e42cfed7ba2551793e15 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 12 Feb 2026 08:18:35 +0100 Subject: [PATCH] updated to v2.0, monorepo customer output --- README.md | 69 ++++++--- customers/demo-felhom.yaml | 2 +- scripts/render.sh | 286 ++++++++++++++++++++++++++----------- 3 files changed, 254 insertions(+), 103 deletions(-) mode change 100755 => 100644 scripts/render.sh diff --git a/README.md b/README.md index fab18ea..c9f0f71 100644 --- a/README.md +++ b/README.md @@ -5,31 +5,44 @@ Central repository for all Felhom customer application deployments. ## Architecture ``` -felhom-app-catalog/ -├── templates/ # Docker Compose templates ({{DOMAIN}} placeholder) +felhom-app-catalog/ ← This repo (source of truth) +├── templates/ # Docker Compose templates with placeholders │ ├── actualbudget/ │ ├── docmost/ │ ├── filebrowser/ │ ├── homebox/ │ ├── immich/ │ ├── mealie/ +│ ├── paperless-ngx/ │ ├── romm/ │ ├── stirling-pdf/ │ └── vaultwarden/ -├── customers/ # Per-customer configuration +├── customers/ # Per-customer configuration (YAML) │ ├── demo-felhom.yaml │ └── pi-customer-1.yaml ├── scripts/ -│ └── render.sh # Renders customer repos from templates -└── README.md +│ └── render.sh # Renders output from templates + customer configs +└── output/ # Generated monorepo (pushed to Gitea) + ├── README.md + ├── demo-felhom/ + │ ├── actualbudget/docker-compose.yml + │ ├── immich/docker-compose.yml + │ └── ... + └── pi-customer-1/ + ├── actualbudget/docker-compose.yml + └── ... ``` +The `output/` directory is what gets pushed to: +**https://gitea.dooplex.hu/admin/customers-felhom.eu** + ## How It Works 1. **Templates** contain Docker Compose files with `{{DOMAIN}}` and `{{HDD_PATH}}` placeholders 2. **Customer configs** define which apps each customer gets, their domain, HDD path, and any version overrides -3. **render.sh** generates per-customer Gitea repos with all placeholders substituted -4. **Portainer GitOps** on each customer node pulls from their repo and deploys +3. **render.sh** substitutes all placeholders and generates the output directory +4. **`--push`** commits and pushes the output to the Gitea monorepo +5. **Portainer GitOps** on each customer node pulls from the same repo, using a different compose path per stack ### Placeholder Reference @@ -43,31 +56,51 @@ felhom-app-catalog/ - **HDD host paths** (`{{HDD_PATH}}/storage/...`): Large user data — photos, documents, ROMs - **Named Docker volumes** (on NVMe): Databases, app config, caches — need fast I/O - Templates that don't use `{{HDD_PATH}}` work without it (e.g. ActualBudget, Mealie) +- If a template needs `{{HDD_PATH}}` but the customer config doesn't set `hdd_path:`, the render script refuses that app and tells you what to fix ## Workflow +### Render & push + +```bash +./scripts/render.sh # Render all customers locally +./scripts/render.sh --push # Render + commit + push to Gitea +./scripts/render.sh --customer demo-felhom # Render one customer only +./scripts/render.sh --dry-run # Preview what would happen +./scripts/render.sh --debug # Verbose output +``` + +The default Gitea repo URL is `https://gitea.dooplex.hu/admin/customers-felhom.eu.git`. +Override with: `GITEA_REPO_URL=https://... ./scripts/render.sh --push` + ### Adding a new app to the catalog -1. Create `templates//docker-compose.yml` using `{{DOMAIN}}` placeholder -2. Add to relevant customer configs in `customers/` -3. Run `./scripts/render.sh` to regenerate customer repos + +1. Create `templates//docker-compose.yml` using `{{DOMAIN}}` and optionally `{{HDD_PATH}}` +2. Add the app name to relevant customer configs in `customers/` +3. Run `./scripts/render.sh --push` ### Updating an app version + 1. Edit the image tag in `templates//docker-compose.yml` -2. Run `./scripts/render.sh` — skips customers with version overrides +2. Run `./scripts/render.sh --push` 3. Portainer auto-detects git changes and redeploys (if polling enabled) +Customers with version overrides keep their pinned version. + ### Adding a new customer -1. Create `customers/.yaml` -2. Create the Gitea repo: `customers/-stacks` -3. Run `./scripts/render.sh --customer ` -4. Set up Portainer GitOps stacks on the customer node + +1. Create `customers/.yaml` (copy an existing one as template) +2. Run `./scripts/render.sh --push` +3. Set up Portainer GitOps stacks on the customer node (see below) ## Portainer Stack Setup (per app) -On the customer's Portainer: +On the customer's Portainer, for each app: + 1. **Stacks → Add Stack → Repository** -2. Repository URL: `https://gitea.felhom.eu/customers/-stacks` -3. Compose path: `/docker-compose.yml` +2. Repository URL: `https://gitea.dooplex.hu/admin/customers-felhom.eu` +3. Compose path: `//docker-compose.yml` + - Example: `demo-felhom/immich/docker-compose.yml` 4. Add environment variables (secrets — DB passwords, API keys, etc.) 5. Enable **GitOps auto-update** (optional, 5-minute polling) 6. Deploy diff --git a/customers/demo-felhom.yaml b/customers/demo-felhom.yaml index 74340c5..daa0a99 100644 --- a/customers/demo-felhom.yaml +++ b/customers/demo-felhom.yaml @@ -1,5 +1,5 @@ # Customer: Demo / Test Server (N100 Mini PC) -# Hardware: Intel N100, 16GB RAM, 512GB NVMe + 1TB HDD +# Hardware: Intel N100, 16GB RAM, 128GB NVMe + 1TB HDD # Network: Local + Cloudflare Tunnel for demo access customer_id: demo-felhom diff --git a/scripts/render.sh b/scripts/render.sh old mode 100755 new mode 100644 index a818de6..64c712c --- a/scripts/render.sh +++ b/scripts/render.sh @@ -1,21 +1,35 @@ #!/bin/bash #=============================================================================== -# Felhom App Catalog - Customer Repo Renderer v1.0 +# Felhom App Catalog - Customer Repo Renderer v2.0 # -# Reads customer YAML configs and generates per-customer stack directories -# by substituting {{DOMAIN}} placeholders in Docker Compose templates. +# Reads customer YAML configs and generates Docker Compose stacks +# by substituting {{DOMAIN}} and {{HDD_PATH}} placeholders in templates. +# +# Output structure (monorepo — all customers in one Gitea repo): +# output/ +# ├── README.md +# ├── demo-felhom/ +# │ ├── actualbudget/docker-compose.yml +# │ ├── immich/docker-compose.yml +# │ └── ... +# └── pi-customer-1/ +# ├── actualbudget/docker-compose.yml +# └── ... +# +# Portainer GitOps compose path example: +# demo-felhom/actualbudget/docker-compose.yml # # Usage: # ./render.sh # Render all customers # ./render.sh --customer demo-felhom # Render specific customer # ./render.sh --dry-run # Show what would be done -# ./render.sh --push # Render + push to Gitea +# ./render.sh --push # Render + commit + push to Gitea # ./render.sh --output-dir /tmp/out # Custom output directory # ./render.sh --debug # Verbose output # # Prerequisites: # - git (for push mode) -# - Access to Gitea (for push mode) +# - Access to Gitea repo (for push mode) # #=============================================================================== @@ -30,9 +44,8 @@ TEMPLATES_DIR="${CATALOG_DIR}/templates" CUSTOMERS_DIR="${CATALOG_DIR}/customers" DEFAULT_OUTPUT_DIR="${CATALOG_DIR}/output" -# Gitea settings (for --push mode) -GITEA_URL="${GITEA_URL:-https://gitea.felhom.eu}" -GITEA_ORG="${GITEA_ORG:-customers}" +# Gitea monorepo settings (for --push mode) +GITEA_REPO_URL="${GITEA_REPO_URL:-https://gitea.dooplex.hu/admin/customers-felhom.eu.git}" #------------------------------------------------------------------------------- # Colors & Logging @@ -53,16 +66,13 @@ log_step() { echo -e "\n${BOLD}[STEP]${NC} $*"; } #------------------------------------------------------------------------------- # Simple YAML parser (no external dependencies) -# Handles our flat customer YAML format #------------------------------------------------------------------------------- yaml_get_value() { - # Get a simple key: value from YAML file local file="$1" key="$2" grep -E "^${key}:" "$file" 2>/dev/null | sed "s/^${key}:[[:space:]]*//" | sed 's/^"//' | sed 's/"$//' | sed "s/^'//" | sed "s/'$//" || echo "" } yaml_get_list() { - # Get a YAML list (items starting with " - ") local file="$1" key="$2" local in_section=false while IFS= read -r line; do @@ -73,19 +83,14 @@ yaml_get_list() { if $in_section; then if [[ "$line" =~ ^[[:space:]]*-[[:space:]]+(.*) ]]; then echo "${BASH_REMATCH[1]}" | sed 's/^"//' | sed 's/"$//' - elif [[ "$line" =~ ^[a-zA-Z_] ]] || [[ -z "$line" && "$in_section" == true ]]; then - # New top-level key or empty line after list = end of section - # But only break on a new top-level key, not empty lines within - if [[ "$line" =~ ^[a-zA-Z_] ]]; then - break - fi + elif [[ "$line" =~ ^[a-zA-Z_] ]]; then + break fi fi done < "$file" } yaml_get_override() { - # Get a specific override value: overrides.key local file="$1" key="$2" local in_overrides=false while IFS= read -r line; do @@ -108,7 +113,7 @@ yaml_get_override() { } #------------------------------------------------------------------------------- -# Template rendering +# Template rendering (one customer) #------------------------------------------------------------------------------- render_customer() { local customer_file="$1" @@ -151,8 +156,11 @@ render_customer() { log_info "HDD path: $hdd_path" fi - # Output directory for this customer - local customer_output="${output_base}/${customer_id}-stacks" + # Output directory: output// (no -stacks suffix) + local customer_output="${output_base}/${customer_id}" + + # Strip .git from repo URL for display + local repo_display="${GITEA_REPO_URL%.git}" if [[ "$DRY_RUN" == "true" ]]; then echo -e " ${CYAN}[DRY-RUN]${NC} Would create: ${customer_output}/" @@ -167,45 +175,48 @@ render_customer() { echo -e " ${YELLOW}[DRY-RUN]${NC} ↳ ⚠ Template uses {{HDD_PATH}} but hdd_path not set!" fi fi - # Show version override if any local version_override version_override=$(yaml_get_override "$customer_file" "${app}_version") if [[ -n "$version_override" ]]; then echo -e " ${CYAN}[DRY-RUN]${NC} ↳ version pinned to: ${version_override}" fi done + echo -e " ${CYAN}[DRY-RUN]${NC} Portainer compose paths:" + for app in "${apps[@]}"; do + echo -e " ${CYAN}[DRY-RUN]${NC} ${customer_id}/${app}/docker-compose.yml" + done return 0 fi # Create output directory mkdir -p "$customer_output" - # Generate a README for the customer repo + # Generate a README for this customer's section cat > "${customer_output}/README.md" << EOF -# ${customer_id} - Application Stacks +# ${customer_id} **Domain:** \`${domain}\` -**HDD Path:** \`${hdd_path:-N/A (no HDD apps)}\` +**HDD Path:** \`${hdd_path:-N/A}\` **Generated:** $(date -u '+%Y-%m-%d %H:%M:%S UTC') -**Source:** felhom-app-catalog (render.sh) ## Deployed Apps -| App | Compose Path | -|-----|-------------| -$(for app in "${apps[@]}"; do echo "| ${app} | \`${app}/docker-compose.yml\` |"; done) +| App | Portainer Compose Path | +|-----|------------------------| +$(for app in "${apps[@]}"; do echo "| ${app} | \`${customer_id}/${app}/docker-compose.yml\` |"; done) -## Portainer Setup +## Portainer Stack Setup For each app, create a stack in Portainer: 1. **Stacks → Add Stack → Repository** -2. Repository URL: \`${GITEA_URL}/${GITEA_ORG}/${customer_id}-stacks\` -3. Compose path: \`/docker-compose.yml\` +2. Repository URL: \`${repo_display}\` +3. Compose path: \`${customer_id}//docker-compose.yml\` 4. Add required environment variables (see comments in compose files) -5. Deploy +5. Enable GitOps auto-update (optional, polling interval: 5 min) +6. Deploy --- -*Auto-generated by felhom-app-catalog. Do not edit manually.* +*Auto-generated by felhom-app-catalog render.sh. Do not edit manually.* EOF # Process each app @@ -232,7 +243,6 @@ EOF rmdir "${customer_output}/${app}" 2>/dev/null || true continue fi - # Remove trailing slash from hdd_path if present local clean_hdd_path="${hdd_path%/}" sed -i "s|{{HDD_PATH}}|${clean_hdd_path}|g" "${customer_output}/${app}/docker-compose.yml" fi @@ -242,20 +252,14 @@ EOF version_override=$(yaml_get_override "$customer_file" "${app}_version") if [[ -n "$version_override" ]]; then log_info " ${app}: applying version override → ${version_override}" - # Find the main image line and replace the tag - # This is a simple approach — works for single-image apps and - # the first image line in multi-service apps case "$app" in immich) - # Immich has server + ML images that should match sed -i "s|ghcr.io/immich-app/immich-server:[^ ]*|ghcr.io/immich-app/immich-server:${version_override}|g" \ "${customer_output}/${app}/docker-compose.yml" sed -i "s|ghcr.io/immich-app/immich-machine-learning:[^ ]*|ghcr.io/immich-app/immich-machine-learning:${version_override}|g" \ "${customer_output}/${app}/docker-compose.yml" ;; *) - # Generic: replace the first image tag (main service) - # Get the first image line and replace its tag local first_image first_image=$(grep -m1 "image:" "${customer_output}/${app}/docker-compose.yml" | sed 's/.*image:[[:space:]]*//') if [[ -n "$first_image" ]]; then @@ -276,40 +280,128 @@ EOF } #------------------------------------------------------------------------------- -# Git push to Gitea +# Git push — single monorepo push (all customers together) #------------------------------------------------------------------------------- push_to_gitea() { - local customer_dir="$1" - local customer_id - customer_id=$(basename "$customer_dir" | sed 's/-stacks$//') + local output_dir="$1" - local repo_url="${GITEA_URL}/${GITEA_ORG}/${customer_id}-stacks.git" + log_step "Pushing to Gitea: ${GITEA_REPO_URL}" - log_step "Pushing to Gitea: ${repo_url}" - - cd "$customer_dir" + cd "$output_dir" + # Initialize git if needed, or update remote if [[ ! -d ".git" ]]; then git init -b main - git remote add origin "$repo_url" + git remote add origin "$GITEA_REPO_URL" + log_info "Initialized new git repo" + else + # Ensure remote URL is current + git remote set-url origin "$GITEA_REPO_URL" 2>/dev/null || \ + git remote add origin "$GITEA_REPO_URL" 2>/dev/null || true fi + # Stage all changes git add -A - if git diff --cached --quiet; then - log_info "No changes to push for ${customer_id}" + + # Check if there are changes to commit + if git diff --cached --quiet 2>/dev/null; then + log_info "No changes to push — output matches Gitea" return 0 fi - git commit -m "Auto-render: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" - - if git push origin main 2>/dev/null; then - log_success "Pushed to ${repo_url}" - else - # Try force push if first time or diverged - log_warn "Normal push failed, trying force push..." - git push -u origin main --force - log_success "Force-pushed to ${repo_url}" + # Show what changed + local changed_files + changed_files=$(git diff --cached --name-only | wc -l) + log_info "Changed files: ${changed_files}" + if [[ "$DEBUG" == "true" ]]; then + git diff --cached --name-only | head -20 fi + + # Commit + local commit_msg="render: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + # Add summary of what customers were rendered + local customers_rendered + customers_rendered=$(find "$output_dir" -maxdepth 1 -mindepth 1 -type d ! -name '.git' -printf '%f ' 2>/dev/null || true) + if [[ -n "$customers_rendered" ]]; then + commit_msg="${commit_msg} [${customers_rendered}]" + fi + + git commit -m "$commit_msg" + + # Push + if git push origin main 2>/dev/null; then + log_success "Pushed to ${GITEA_REPO_URL}" + else + log_warn "Normal push failed — trying pull+rebase first..." + if git pull --rebase origin main 2>/dev/null; then + git push origin main + log_success "Rebased and pushed to ${GITEA_REPO_URL}" + else + log_warn "Rebase failed — force pushing..." + git push -u origin main --force + log_success "Force-pushed to ${GITEA_REPO_URL}" + fi + fi +} + +#------------------------------------------------------------------------------- +# Generate root README for the monorepo +#------------------------------------------------------------------------------- +generate_root_readme() { + local output_dir="$1" + shift + local customer_files=("$@") + + local repo_display="${GITEA_REPO_URL%.git}" + + cat > "${output_dir}/README.md" << 'HEADER' +# Felhom Customer Stacks + +Auto-generated Docker Compose stacks for all Felhom customers. +**Do not edit manually** — regenerate with `render.sh` from `felhom-app-catalog`. + +HEADER + + echo "## Customers" >> "${output_dir}/README.md" + echo "" >> "${output_dir}/README.md" + echo "| Customer | Domain | Hardware | Apps |" >> "${output_dir}/README.md" + echo "|----------|--------|----------|------|" >> "${output_dir}/README.md" + + for cf in "${customer_files[@]}"; do + local cid cdomain chw capps + cid=$(yaml_get_value "$cf" "customer_id") + cdomain=$(yaml_get_value "$cf" "domain") + chw=$(yaml_get_value "$cf" "hardware") + # Count apps + capps=$(yaml_get_list "$cf" "apps" | wc -l) + echo "| [${cid}](./${cid}/) | \`${cdomain}\` | ${chw} | ${capps} apps |" >> "${output_dir}/README.md" + done + + cat >> "${output_dir}/README.md" << EOF + +## Portainer GitOps Setup + +For each app stack on a customer node: +1. **Stacks → Add Stack → Repository** +2. Repository URL: \`${repo_display}\` +3. Compose path: \`//docker-compose.yml\` +4. Set environment variables (secrets — see comments in compose files) +5. Optional: Enable auto-update polling (5 min) + +## Regenerating + +From the \`felhom-app-catalog\` repo: +\`\`\`bash +./scripts/render.sh # Render locally +./scripts/render.sh --push # Render + push to Gitea +./scripts/render.sh --dry-run # Preview changes +\`\`\` + +--- +*Last generated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')* +EOF + + log_debug "Generated root README.md" } #------------------------------------------------------------------------------- @@ -323,30 +415,32 @@ OUTPUT_DIR="" print_help() { cat << 'EOF' -Felhom App Catalog - Customer Repo Renderer v1.0 +Felhom App Catalog - Customer Repo Renderer v2.0 -Generates per-customer Docker Compose stacks from templates. +Renders Docker Compose stacks from templates into a single Gitea monorepo. +All customers share one repo, each in their own subdirectory. USAGE: ./render.sh [OPTIONS] OPTIONS: - --customer ID Render only this customer + --customer ID Render only this customer (others left untouched) --output-dir DIR Output directory (default: ./output/) - --push Push rendered repos to Gitea + --push Render + commit + push to Gitea --dry-run Show what would be done --debug Verbose output -h, --help Show this help ENVIRONMENT VARIABLES: - GITEA_URL Gitea base URL (default: https://gitea.felhom.eu) - GITEA_ORG Gitea organization (default: customers) + GITEA_REPO_URL Full Gitea repo URL + (default: https://gitea.dooplex.hu/admin/customers-felhom.eu.git) EXAMPLES: ./render.sh # Render all customers ./render.sh --customer demo-felhom # Render one customer ./render.sh --dry-run # Preview changes ./render.sh --push # Render + push to Gitea + GITEA_REPO_URL=https://gitea.example.com/org/repo.git ./render.sh --push EOF } @@ -369,13 +463,14 @@ OUTPUT_DIR="${OUTPUT_DIR:-${DEFAULT_OUTPUT_DIR}}" #------------------------------------------------------------------------------- main() { echo "" - echo -e "${BOLD}Felhom App Catalog — Renderer${NC}" - echo -e "Templates: ${TEMPLATES_DIR}" - echo -e "Customers: ${CUSTOMERS_DIR}" - echo -e "Output: ${OUTPUT_DIR}" + echo -e "${BOLD}Felhom App Catalog — Renderer v2.0${NC}" + echo -e "Templates: ${TEMPLATES_DIR}" + echo -e "Customers: ${CUSTOMERS_DIR}" + echo -e "Output: ${OUTPUT_DIR}" + echo -e "Gitea repo: ${GITEA_REPO_URL}" echo "" - # Validate directories exist + # Validate directories if [[ ! -d "$TEMPLATES_DIR" ]]; then log_error "Templates directory not found: $TEMPLATES_DIR" exit 1 @@ -412,35 +507,58 @@ main() { mkdir -p "$OUTPUT_DIR" fi + # If --push and output dir already has a .git from previous push, pull first + if [[ "$PUSH" == "true" && "$DRY_RUN" != "true" && -d "${OUTPUT_DIR}/.git" ]]; then + log_info "Pulling latest from Gitea before rendering..." + cd "$OUTPUT_DIR" + git pull --rebase origin main 2>/dev/null || true + cd - > /dev/null + fi + # Render each customer local success_count=0 for customer_file in "${customer_files[@]}"; do if render_customer "$customer_file" "$OUTPUT_DIR"; then success_count=$((success_count + 1)) - - # Push to Gitea if requested - if [[ "$PUSH" == "true" && "$DRY_RUN" != "true" ]]; then - local cid - cid=$(yaml_get_value "$customer_file" "customer_id") - push_to_gitea "${OUTPUT_DIR}/${cid}-stacks" - fi fi done + # Generate root README (only when rendering all customers) + if [[ -z "$TARGET_CUSTOMER" && "$DRY_RUN" != "true" ]]; then + # Get ALL customer files for the root README (even if we only rendered some) + local all_customer_files=() + while IFS= read -r -d '' f; do + all_customer_files+=("$f") + done < <(find "$CUSTOMERS_DIR" -name "*.yaml" -print0 | sort -z) + generate_root_readme "$OUTPUT_DIR" "${all_customer_files[@]}" + fi + echo "" echo -e "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BOLD}${GREEN} Rendering complete: ${success_count}/${#customer_files[@]} customers${NC}" echo -e "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo "" if [[ "$DRY_RUN" != "true" ]]; then - echo -e "${BOLD}Output:${NC}" - find "$OUTPUT_DIR" -name "docker-compose.yml" -printf " %P\n" | sort + echo "" + echo -e "${BOLD}Output structure:${NC}" + # Show tree-like output + for d in "${OUTPUT_DIR}"/*/; do + [[ -d "$d" && "$(basename "$d")" != ".git" ]] || continue + local cid + cid=$(basename "$d") + local app_count + app_count=$(find "$d" -name "docker-compose.yml" | wc -l) + echo -e " 📁 ${BOLD}${cid}/${NC} (${app_count} apps)" + find "$d" -name "docker-compose.yml" -printf " %P\n" | sort + done echo "" fi - if [[ "$PUSH" != "true" && "$DRY_RUN" != "true" ]]; then - echo -e "${YELLOW}Tip:${NC} Use --push to automatically push to Gitea repos" + # Push all at once + if [[ "$PUSH" == "true" && "$DRY_RUN" != "true" ]]; then + push_to_gitea "$OUTPUT_DIR" + elif [[ "$PUSH" != "true" && "$DRY_RUN" != "true" ]]; then + echo -e "${YELLOW}Tip:${NC} Use --push to commit and push to Gitea" fi }