#!/usr/bin/env bash # build-golden.sh — build the Felhom golden base LXC archive (slice 7). # # Produces a minimal Debian + Docker, unprivileged, nesting=1,keyctl=1, overlayfs LXC, baked # identity-clean, and archives it for a token-restore by the bring-up reconcile job # (internal/reconcile/bringup.go). Run as root@pam on a Proxmox host (the keyctl=1 feature flag # is root-only — phase3 #1; this is the ONE root step, off the per-customer path). # # Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993): # - F3: removing the SSH host keys does NOT auto-regenerate them on Debian (pct restore runs no # keygen hook), so a baked, Condition-gated first-boot unit regenerates them — keeping the # agent's front half host-side-only. The gate (ConditionPathExists=!…) makes it fire on a # provision (golden, keys absent) and no-op on a DR restore (customer backup, keys present), # symmetric with machine-id. # - machine-id: truncated; systemd regenerates it on first boot for free (no unit needed). # # Slice 8A — the golden now also BAKES the in-guest controller (decision: image baked at golden # build on the trusted host, so NO registry credential ever enters a customer guest at deploy) and # a controller-bootstrap unit that, on boot, deploys the baked image from the agent-populated # config mount (/etc/felhom-bootstrap/bootstrap.json) — no docker login/pull at deploy. Refreshing # the golden bumps the controller baseline; controller self-update covers in-between drift. # # Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] [CONTROLLER_IMAGE] # Build-time registry login for the controller pull (used ONCE inside the build guest, then logged # out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment. set -euo pipefail VMID="${1:-9100}" TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}" ROOTFS_STORAGE="${3:-local-lvm}" ARCHIVE_STORAGE="${4:-local}" BRIDGE="${5:-vmbr0}" CONTROLLER_IMAGE="${6:-gitea.dooplex.hu/admin/felhom-controller:0.43.0}" REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}" echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged) …" pct create "$VMID" "$TEMPLATE" \ --hostname felhom-golden --unprivileged 1 \ --features nesting=1,keyctl=1 \ --rootfs "${ROOTFS_STORAGE}:8" --cores 2 --memory 2048 \ --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0 echo "[golden] starting + installing Docker (official repo, trixie channel) …" pct start "$VMID" # wait for DHCP/DNS for i in $(seq 1 30); do if pct exec "$VMID" -- getent hosts download.docker.com >/dev/null 2>&1; then break; fi sleep 1 done pct exec "$VMID" -- bash -c ' set -e export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq ca-certificates curl >/dev/null install -m0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" \ > /etc/apt/sources.list.d/docker.list apt-get update -qq apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null ' echo "[golden] verifying Docker works in the build guest …" pct exec "$VMID" -- bash -c 'systemctl start docker; sleep 2; docker run --rm hello-world >/dev/null && echo " docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"))"' echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …" # docker login is used ONCE here on the trusted build host, then logged out before archiving so # the credential is NEVER baked into the golden. The IMAGE is what gets baked (in Docker storage). if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null" fi pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'" pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json" # Record the baked image ref for the bootstrap unit (so the unit needs no login/pull). pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image" # Bake the base-infrastructure images (traefik, cloudflared, filebrowser) so the controller's # first-boot bring-up (EnsureBaseStack) is OFFLINE-capable — no registry pull at deploy. These are # PUBLIC Docker Hub images (no cred needed). The PINNED tags MUST match the controller's # internal/infra constants (TraefikImage / CloudflaredImage / FileBrowserImage); a drift means the # golden bakes one image and the controller requests another (→ a pull at deploy, defeating the goal). INFRA_IMAGES=( "traefik:v3.6.7" "cloudflare/cloudflared:2026.6.0" "gtstef/filebrowser:1.3.3-stable" ) echo "[golden] baking base-infra images: ${INFRA_IMAGES[*]} …" for img in "${INFRA_IMAGES[@]}"; do # Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails # mid-bake with a confusing error). pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \ || { echo "[golden] FATAL: pinned base-infra image does not resolve: $img"; exit 1; } pct exec "$VMID" -- bash -c "docker pull '$img'" done echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …" pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH' #!/bin/bash # felhom controller-bootstrap (slice 8A): the host agent's back-half populated the read-only # config mount /etc/felhom-bootstrap; this golden-baked oneshot deploys the BAKED controller image # with that config. NO docker login / NO docker pull — the image is already in this golden's Docker # storage (and self-update handles version drift). Host-side only; the agent never enters the guest. set -euo pipefail CFG=/etc/felhom-bootstrap/bootstrap.json [ -r "$CFG" ] || { echo "[ctrl-bootstrap] no $CFG — not provisioned, nothing to do"; exit 0; } IMAGE=$(cat /etc/felhom-controller-image 2>/dev/null || true) [ -n "$IMAGE" ] || { echo "[ctrl-bootstrap] FATAL: /etc/felhom-controller-image missing"; exit 1; } # Per-guest container hostname (slice base-infra/3A): derive from the bootstrap's customer.id so the # controller's os.Hostname() (its hub-reported hostname) is the customer id, not the Docker container # ID. Portable, dependency-free parse (NO jq in the golden) — bootstrap.json has exactly one "id" key # (customer.id). Falls back to no --hostname if the parse yields nothing (fail-safe). CUSTOMER_ID=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CFG" | head -1) # SECURITY: $CUSTOMER_ID is interpolated into `docker run` — reject anything that isn't a DNS-safe # label so a malformed/hostile customer.id can't smuggle extra docker flags (e.g. --privileged, -v). # Then pass it via a quoted array (never word-split). Empty/invalid → no --hostname (fail-safe). case "$CUSTOMER_ID" in ""|*[!a-zA-Z0-9._-]*|-*) CUSTOMER_ID="" ;; esac HOSTNAME_ARGS=() [ -n "$CUSTOMER_ID" ] && HOSTNAME_ARGS=(--hostname "$CUSTOMER_ID") echo "[ctrl-bootstrap] deploying $IMAGE from $CFG (hostname=${CUSTOMER_ID:-})" docker rm -f felhom-controller >/dev/null 2>&1 || true # Section-G fix (base-infra slice): the controller writes app/infra compose stacks under # /opt/docker/stacks INSIDE the container, but `docker compose up` is executed by the GUEST daemon # (shared socket), which resolves every relative bind source on the GUEST filesystem. Without a # SAME-PATH host bind for /opt/docker/stacks, the daemon can't see those bind sources and silently # creates empty dirs → every bind-mounted stack (base infra AND customer apps) breaks. A named volume # would NOT fix this (it resolves to /var/lib/docker/volumes/...). Pre-create the dir, then same-path bind. mkdir -p /opt/docker/stacks # slice 8C: the controller is DE-PRIVILEGED — disk execution (scan/format/mount/migrate) is the # host agent's job now, so this run grants NO disk privileges: no --privileged, no /dev, no # /etc/fstab, no rshared /mnt. Only the bootstrap config (ro), the data volume, the stacks dir # (same-path host bind), and the docker socket (app/stack management). The controller reaches the # agent's local API for disk management. docker run -d --name felhom-controller --restart unless-stopped "${HOSTNAME_ARGS[@]}" \ -e FELHOM_BOOTSTRAP_PATH=/etc/felhom-bootstrap/bootstrap.json \ -v /etc/felhom-bootstrap:/etc/felhom-bootstrap:ro \ -v felhom-controller-data:/opt/docker/felhom-controller \ -v /opt/docker/stacks:/opt/docker/stacks \ -v /var/run/docker.sock:/var/run/docker.sock \ "$IMAGE" echo "[ctrl-bootstrap] controller started" BOOTSH pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.service < /etc/systemd/system/felhom-regen-hostkeys.service </dev/null || true apt-get clean; rm -rf /var/lib/apt/lists/* rm -f /etc/ssh/ssh_host_* # regenerated on first boot by the baked unit (F3) truncate -s 0 /etc/machine-id # systemd regenerates on first boot (free) rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id rm -rf /var/log/*; : > /root/.bash_history rm -f /etc/hostname # set per-guest at provision (host-side token config) ' echo "[golden] stop + archive …" pct stop "$VMID" vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1) echo "[golden] DONE. golden archive volid: ${VOLID:-}" echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"