197 lines
11 KiB
Bash
197 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
# build-golden.sh — build the Felhom golden base LXC archive (slice 7).
|
|
#
|
|
# Produces a minimal Debian + Docker, unprivileged, nesting=1,keyctl=1, overlayfs LXC, baked
|
|
# identity-clean, and archives it for a token-restore by the bring-up reconcile job
|
|
# (internal/reconcile/bringup.go). Run as root@pam on a Proxmox host (the keyctl=1 feature flag
|
|
# is root-only — phase3 #1; this is the ONE root step, off the per-customer path).
|
|
#
|
|
# Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993):
|
|
# - F3: removing the SSH host keys does NOT auto-regenerate them on Debian (pct restore runs no
|
|
# keygen hook), so a baked, Condition-gated first-boot unit regenerates them — keeping the
|
|
# agent's front half host-side-only. The gate (ConditionPathExists=!…) makes it fire on a
|
|
# provision (golden, keys absent) and no-op on a DR restore (customer backup, keys present),
|
|
# symmetric with machine-id.
|
|
# - machine-id: truncated; systemd regenerates it on first boot for free (no unit needed).
|
|
#
|
|
# Slice 8A — the golden now also BAKES the in-guest controller (decision: image baked at golden
|
|
# build on the trusted host, so NO registry credential ever enters a customer guest at deploy) and
|
|
# a controller-bootstrap unit that, on boot, deploys the baked image from the agent-populated
|
|
# config mount (/etc/felhom-bootstrap/bootstrap.json) — no docker login/pull at deploy. Refreshing
|
|
# the golden bumps the controller baseline; controller self-update covers in-between drift.
|
|
#
|
|
# Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] [CONTROLLER_IMAGE]
|
|
# Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
|
|
# out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
|
|
set -euo pipefail
|
|
|
|
VMID="${1:-9100}"
|
|
TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}"
|
|
ROOTFS_STORAGE="${3:-local-lvm}"
|
|
ARCHIVE_STORAGE="${4:-local}"
|
|
BRIDGE="${5:-vmbr0}"
|
|
CONTROLLER_IMAGE="${6:-gitea.dooplex.hu/admin/felhom-controller:0.42.1}"
|
|
REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
|
|
|
|
echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged) …"
|
|
pct create "$VMID" "$TEMPLATE" \
|
|
--hostname felhom-golden --unprivileged 1 \
|
|
--features nesting=1,keyctl=1 \
|
|
--rootfs "${ROOTFS_STORAGE}:8" --cores 2 --memory 2048 \
|
|
--net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0
|
|
|
|
echo "[golden] starting + installing Docker (official repo, trixie channel) …"
|
|
pct start "$VMID"
|
|
# wait for DHCP/DNS
|
|
for i in $(seq 1 30); do
|
|
if pct exec "$VMID" -- getent hosts download.docker.com >/dev/null 2>&1; then break; fi
|
|
sleep 1
|
|
done
|
|
pct exec "$VMID" -- bash -c '
|
|
set -e
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -qq
|
|
apt-get install -y -qq ca-certificates curl >/dev/null
|
|
install -m0755 -d /etc/apt/keyrings
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" \
|
|
> /etc/apt/sources.list.d/docker.list
|
|
apt-get update -qq
|
|
apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
|
|
'
|
|
echo "[golden] verifying Docker works in the build guest …"
|
|
pct exec "$VMID" -- bash -c 'systemctl start docker; sleep 2; docker run --rm hello-world >/dev/null && echo " docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"))"'
|
|
|
|
echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
|
|
# docker login is used ONCE here on the trusted build host, then logged out before archiving so
|
|
# the credential is NEVER baked into the golden. The IMAGE is what gets baked (in Docker storage).
|
|
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
|
|
pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null"
|
|
fi
|
|
pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'"
|
|
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
|
|
# Record the baked image ref for the bootstrap unit (so the unit needs no login/pull).
|
|
pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image"
|
|
|
|
# Bake the base-infrastructure images (traefik, cloudflared, filebrowser) so the controller's
|
|
# first-boot bring-up (EnsureBaseStack) is OFFLINE-capable — no registry pull at deploy. These are
|
|
# PUBLIC Docker Hub images (no cred needed). The PINNED tags MUST match the controller's
|
|
# internal/infra constants (TraefikImage / CloudflaredImage / FileBrowserImage); a drift means the
|
|
# golden bakes one image and the controller requests another (→ a pull at deploy, defeating the goal).
|
|
INFRA_IMAGES=(
|
|
"traefik:v3.6.7"
|
|
"cloudflare/cloudflared:2026.6.0"
|
|
"gtstef/filebrowser:1.3.3-stable"
|
|
)
|
|
echo "[golden] baking base-infra images: ${INFRA_IMAGES[*]} …"
|
|
for img in "${INFRA_IMAGES[@]}"; do
|
|
# Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails
|
|
# mid-bake with a confusing error).
|
|
pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \
|
|
|| { echo "[golden] FATAL: pinned base-infra image does not resolve: $img"; exit 1; }
|
|
pct exec "$VMID" -- bash -c "docker pull '$img'"
|
|
done
|
|
|
|
echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …"
|
|
pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH'
|
|
#!/bin/bash
|
|
# felhom controller-bootstrap (slice 8A): the host agent's back-half populated the read-only
|
|
# config mount /etc/felhom-bootstrap; this golden-baked oneshot deploys the BAKED controller image
|
|
# with that config. NO docker login / NO docker pull — the image is already in this golden's Docker
|
|
# storage (and self-update handles version drift). Host-side only; the agent never enters the guest.
|
|
set -euo pipefail
|
|
CFG=/etc/felhom-bootstrap/bootstrap.json
|
|
[ -r "$CFG" ] || { echo "[ctrl-bootstrap] no $CFG — not provisioned, nothing to do"; exit 0; }
|
|
IMAGE=$(cat /etc/felhom-controller-image 2>/dev/null || true)
|
|
[ -n "$IMAGE" ] || { echo "[ctrl-bootstrap] FATAL: /etc/felhom-controller-image missing"; exit 1; }
|
|
# Per-guest container hostname (slice base-infra/3A): derive from the bootstrap's customer.id so the
|
|
# controller's os.Hostname() (its hub-reported hostname) is the customer id, not the Docker container
|
|
# ID. Portable, dependency-free parse (NO jq in the golden) — bootstrap.json has exactly one "id" key
|
|
# (customer.id). Falls back to no --hostname if the parse yields nothing (fail-safe).
|
|
CUSTOMER_ID=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CFG" | head -1)
|
|
# SECURITY: $CUSTOMER_ID is interpolated into `docker run` — reject anything that isn't a DNS-safe
|
|
# label so a malformed/hostile customer.id can't smuggle extra docker flags (e.g. --privileged, -v).
|
|
# Then pass it via a quoted array (never word-split). Empty/invalid → no --hostname (fail-safe).
|
|
case "$CUSTOMER_ID" in
|
|
""|*[!a-zA-Z0-9._-]*|-*) CUSTOMER_ID="" ;;
|
|
esac
|
|
HOSTNAME_ARGS=()
|
|
[ -n "$CUSTOMER_ID" ] && HOSTNAME_ARGS=(--hostname "$CUSTOMER_ID")
|
|
echo "[ctrl-bootstrap] deploying $IMAGE from $CFG (hostname=${CUSTOMER_ID:-<unset>})"
|
|
docker rm -f felhom-controller >/dev/null 2>&1 || true
|
|
# Section-G fix (base-infra slice): the controller writes app/infra compose stacks under
|
|
# /opt/docker/stacks INSIDE the container, but `docker compose up` is executed by the GUEST daemon
|
|
# (shared socket), which resolves every relative bind source on the GUEST filesystem. Without a
|
|
# SAME-PATH host bind for /opt/docker/stacks, the daemon can't see those bind sources and silently
|
|
# creates empty dirs → every bind-mounted stack (base infra AND customer apps) breaks. A named volume
|
|
# would NOT fix this (it resolves to /var/lib/docker/volumes/...). Pre-create the dir, then same-path bind.
|
|
mkdir -p /opt/docker/stacks
|
|
# slice 8C: the controller is DE-PRIVILEGED — disk execution (scan/format/mount/migrate) is the
|
|
# host agent's job now, so this run grants NO disk privileges: no --privileged, no /dev, no
|
|
# /etc/fstab, no rshared /mnt. Only the bootstrap config (ro), the data volume, the stacks dir
|
|
# (same-path host bind), and the docker socket (app/stack management). The controller reaches the
|
|
# agent's local API for disk management.
|
|
docker run -d --name felhom-controller --restart unless-stopped "${HOSTNAME_ARGS[@]}" \
|
|
-e FELHOM_BOOTSTRAP_PATH=/etc/felhom-bootstrap/bootstrap.json \
|
|
-v /etc/felhom-bootstrap:/etc/felhom-bootstrap:ro \
|
|
-v felhom-controller-data:/opt/docker/felhom-controller \
|
|
-v /opt/docker/stacks:/opt/docker/stacks \
|
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
|
"$IMAGE"
|
|
echo "[ctrl-bootstrap] controller started"
|
|
BOOTSH
|
|
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.service <<UNIT
|
|
[Unit]
|
|
Description=Felhom controller bootstrap (deploy the baked controller from the agent-populated config mount)
|
|
After=docker.service network-online.target
|
|
Wants=docker.service network-online.target
|
|
ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=yes
|
|
ExecStart=/usr/local/sbin/felhom-controller-bootstrap.sh
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
systemctl enable felhom-controller-bootstrap.service'
|
|
|
|
echo "[golden] baking the first-boot SSH host-key regeneration unit (F3) …"
|
|
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-regen-hostkeys.service <<UNIT
|
|
[Unit]
|
|
Description=Regenerate SSH host keys on first boot if absent
|
|
ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
|
|
DefaultDependencies=no
|
|
After=local-fs.target
|
|
Before=ssh.service sshd.service
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=yes
|
|
ExecStart=/usr/bin/ssh-keygen -A
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
systemctl enable felhom-regen-hostkeys.service'
|
|
|
|
echo "[golden] identity-clean + minimize …"
|
|
pct exec "$VMID" -- bash -c '
|
|
systemctl stop docker containerd 2>/dev/null || true
|
|
apt-get clean; rm -rf /var/lib/apt/lists/*
|
|
rm -f /etc/ssh/ssh_host_* # regenerated on first boot by the baked unit (F3)
|
|
truncate -s 0 /etc/machine-id # systemd regenerates on first boot (free)
|
|
rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id
|
|
rm -rf /var/log/*; : > /root/.bash_history
|
|
rm -f /etc/hostname # set per-guest at provision (host-side token config)
|
|
'
|
|
|
|
echo "[golden] stop + archive …"
|
|
pct stop "$VMID"
|
|
vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd
|
|
|
|
VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
|
|
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>}"
|
|
echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"
|