felhom-agent/configs/build-golden.sh

#!/usr/bin/env bash
# build-golden.sh — build the Felhom golden base LXC archive (slice 7).
#
# Produces a minimal Debian + Docker, unprivileged, nesting=1,keyctl=1, overlayfs LXC, baked
# identity-clean, and archives it for a token-restore by the bring-up reconcile job
# (internal/reconcile/bringup.go). Run as root@pam on a Proxmox host (the keyctl=1 feature flag
# is root-only — phase3 #1; this is the ONE root step, off the per-customer path).
#
# Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993):
#   - F3: removing the SSH host keys does NOT auto-regenerate them on Debian (pct restore runs no
#     keygen hook), so a baked, Condition-gated first-boot unit regenerates them — keeping the
#     agent's front half host-side-only. The gate (ConditionPathExists=!…) makes it fire on a
#     provision (golden, keys absent) and no-op on a DR restore (customer backup, keys present),
#     symmetric with machine-id.
#   - machine-id: truncated; systemd regenerates it on first boot for free (no unit needed).
#
# Slice 8A — the golden now also BAKES the in-guest controller (decision: image baked at golden
# build on the trusted host, so NO registry credential ever enters a customer guest at deploy) and
# a controller-bootstrap unit that, on boot, deploys the baked image from the agent-populated
# config mount (/etc/felhom-bootstrap/bootstrap.json) — no docker login/pull at deploy. Refreshing
# the golden bumps the controller baseline; controller self-update covers in-between drift.
#
# Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] [CONTROLLER_IMAGE]
# Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
# out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
#
# OS / Docker-data SPLIT (storage-split slice): the golden is built with a SMALL OS rootfs and a
# SEPARATE Docker-data volume mounted at /var/lib/docker (mp0, backup=1). The baked controller +
# infra images land on that volume and travel INSIDE the golden archive — so provisioned guests boot
# from baked images with no registry pull. The split is for RESILIENCE: an isolated OS rootfs stays
# bootable + agent-recoverable if the Docker volume fills (the controller's prevention layer keeps it
# from filling). Sizes are env-overridable (OS_SIZE_GB / GOLDEN_DOCKER_GB); provision GROWS the data
# volume to the per-customer target (bringup.go DataVolGrowGB). backup=1 is MANDATORY on the data mp:
# without it vzdump EXCLUDES the volume (extra LXC mountpoints default backup=0 — storage-split B3),
# so the archive would carry NO images and provisioned guests would boot imageless.
set -euo pipefail

VMID="${1:-9100}"
TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}"
ROOTFS_STORAGE="${3:-local-lvm}"
ARCHIVE_STORAGE="${4:-local}"
BRIDGE="${5:-vmbr0}"
# Default to the CURRENT controller so an argument-less build is never wildly stale. ALWAYS pass the
# controller version explicitly at each rebuild (this default only bounds the worst case); a future
# `make golden` that resolves the latest pullable tag would remove the need for a hand-bumped default.
CONTROLLER_IMAGE="${6:-gitea.dooplex.hu/admin/felhom-controller:0.85.1}"
REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
# OS rootfs size (GiB) and the golden's Docker-data volume size (GiB). Keep GOLDEN_DOCKER_GB just
# large enough for the baked images + headroom; provision grows it to the per-customer target.
OS_SIZE_GB="${OS_SIZE_GB:-32}"
GOLDEN_DOCKER_GB="${GOLDEN_DOCKER_GB:-16}"
# The golden's SSD user-data volume (GiB) mounted at /mnt/sys_drive (mp1, backup=1) — the controller's
# system_data_path. Ships small + near-empty (the controller creates <sys_drive>/felhom-data itself once
# it's a real mountpoint); provision GROWS it to the per-customer target (bringup.go SysDataGrowGB). Like
# mp0, backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra mountpoints default backup=0 —
# storage-split B3) and the user-data area would silently fall out of PBS coverage.
GOLDEN_SYSDATA_GB="${GOLDEN_SYSDATA_GB:-8}"

echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G @ /var/lib/docker + user-data ${GOLDEN_SYSDATA_GB}G @ /mnt/sys_drive, both backup=1) …"
pct create "$VMID" "$TEMPLATE" \
  --hostname felhom-golden --unprivileged 1 \
  --features nesting=1,keyctl=1 \
  --rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \
  --mp0 "${ROOTFS_STORAGE}:${GOLDEN_DOCKER_GB},mp=/var/lib/docker,backup=1" \
  --mp1 "${ROOTFS_STORAGE}:${GOLDEN_SYSDATA_GB},mp=/mnt/sys_drive,backup=1" \
  --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0

echo "[golden] starting + installing Docker (official repo, trixie channel) …"
pct start "$VMID"
# wait for DHCP/DNS
for i in $(seq 1 30); do
  if pct exec "$VMID" -- getent hosts download.docker.com >/dev/null 2>&1; then break; fi
  sleep 1
done
pct exec "$VMID" -- bash -c '
  set -e
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -qq
  apt-get install -y -qq ca-certificates curl >/dev/null
  install -m0755 -d /etc/apt/keyrings
  curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
  echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" \
    > /etc/apt/sources.list.d/docker.list
  apt-get update -qq
  apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
'
echo "[golden] baking daemon.json: classic overlay2 driver (containerd-snapshotter OFF) + log rotation …"
# containerd-snapshotter (Docker 28+/29 default) keeps the IMAGE content store under
# /var/lib/containerd — which is NOT /var/lib/docker, so it would stay on the OS rootfs and the split
# would only move named volumes, defeating the whole point (validated: images landed on the rootfs).
# The classic overlay2 driver stores EVERYTHING (images + overlay + volumes) under data-root
# (/var/lib/docker) = the data volume, which is exactly what "one data-root = one partition for all
# images + overlay" requires. It also makes the controller's statfs("/") (its overlay root) report the
# DATA volume, which the prevention layer depends on. /var/lib/docker is the mp0 mount (mounted empty
# before docker installs), so data-root needs no override. Log caps kill the most common runaway.
pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json <<JSON
{
  "features": { "containerd-snapshotter": false },
  "log-driver": "json-file",
  "log-opts": { "max-size": "10m", "max-file": "3" }
}
JSON'
echo "[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) …"
# RESTART (not start): docker-ce auto-starts on install with the DEFAULT config, so it is already
# running by now; only a restart picks up the daemon.json just written (overlay2 + log caps).
pct exec "$VMID" -- bash -c 'systemctl restart docker; sleep 3; docker run --rm hello-world >/dev/null && echo "  docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); data-root $(docker info 2>/dev/null | sed -n "s/.*Docker Root Dir: //p"))"'
# Guard: the image store MUST be on the data volume now. /var/lib/containerd holding the images would
# mean containerd-snapshotter is still on (the split would leave images on the rootfs).
pct exec "$VMID" -- bash -c 'drv=$(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); [ "$drv" = "overlay2" ] || { echo "[golden] FATAL: storage driver is $drv, expected overlay2 — images would not land on the data volume"; exit 1; }'
# Confirm /var/lib/docker is genuinely the dedicated volume, not the rootfs (catch a silent mp miss).
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo "  /var/lib/docker is a separate mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker)" || { echo "[golden] FATAL: /var/lib/docker is NOT a separate mount — the mp0 split did not take"; exit 1; }'
# Same guard for the SSD user-data volume (mp1): /mnt/sys_drive must be its own mount, not the rootfs
# device — otherwise the controller's system_data_path lands on the OS drive and it warns (the whole
# point of this volume is to clear that warning).
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo "  /mnt/sys_drive is a separate mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a separate mount — the mp1 split did not take"; exit 1; }'

echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
# docker login is used ONCE here on the trusted build host, then logged out before archiving so
# the credential is NEVER baked into the golden. The IMAGE is what gets baked (in Docker storage).
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
  pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null"
fi
pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'"
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
# Record the baked image ref for the bootstrap unit (so the unit needs no login/pull).
pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image"

# Bake the base-infrastructure images (traefik, cloudflared, filebrowser) so the controller's
# first-boot bring-up (EnsureBaseStack) is OFFLINE-capable — no registry pull at deploy. These are
# PUBLIC Docker Hub images (no cred needed). The PINNED tags MUST match the controller's
# internal/infra constants (TraefikImage / CloudflaredImage / FileBrowserImage); a drift means the
# golden bakes one image and the controller requests another (→ a pull at deploy, defeating the goal).
INFRA_IMAGES=(
  "traefik:v3.6.7"
  "cloudflare/cloudflared:2026.6.0"
  "gtstef/filebrowser:1.3.3-stable"
)
echo "[golden] baking base-infra images: ${INFRA_IMAGES[*]} …"
for img in "${INFRA_IMAGES[@]}"; do
  # Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails
  # mid-bake with a confusing error).
  pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \
    || { echo "[golden] FATAL: pinned base-infra image does not resolve: $img"; exit 1; }
  pct exec "$VMID" -- bash -c "docker pull '$img'"
done

echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …"
pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH'
#!/bin/bash
# felhom controller-bootstrap (slice 8A): the host agent's back-half populated the read-only
# config mount /etc/felhom-bootstrap; this golden-baked oneshot deploys the BAKED controller image
# with that config. NO docker login / NO docker pull — the image is already in this golden's Docker
# storage (and self-update handles version drift). Host-side only; the agent never enters the guest.
set -euo pipefail
CFG=/etc/felhom-bootstrap/bootstrap.json
[ -r "$CFG" ] || { echo "[ctrl-bootstrap] no $CFG — not provisioned, nothing to do"; exit 0; }
IMAGE=$(cat /etc/felhom-controller-image 2>/dev/null || true)
[ -n "$IMAGE" ] || { echo "[ctrl-bootstrap] FATAL: /etc/felhom-controller-image missing"; exit 1; }
# Per-guest container hostname (slice base-infra/3A): derive from the bootstrap's customer.id so the
# controller's os.Hostname() (its hub-reported hostname) is the customer id, not the Docker container
# ID. Portable, dependency-free parse (NO jq in the golden) — bootstrap.json has exactly one "id" key
# (customer.id). Falls back to no --hostname if the parse yields nothing (fail-safe).
CUSTOMER_ID=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CFG" | head -1)
# SECURITY: $CUSTOMER_ID is interpolated into `docker run` — reject anything that isn't a DNS-safe
# label so a malformed/hostile customer.id can't smuggle extra docker flags (e.g. --privileged, -v).
# Then pass it via a quoted array (never word-split). Empty/invalid → no --hostname (fail-safe).
case "$CUSTOMER_ID" in
  ""|*[!a-zA-Z0-9._-]*|-*) CUSTOMER_ID="" ;;
esac
HOSTNAME_ARGS=()
[ -n "$CUSTOMER_ID" ] && HOSTNAME_ARGS=(--hostname "$CUSTOMER_ID")
echo "[ctrl-bootstrap] deploying $IMAGE from $CFG (hostname=${CUSTOMER_ID:-<unset>})"
docker rm -f felhom-controller >/dev/null 2>&1 || true
# Section-G fix (base-infra slice): the controller writes app/infra compose stacks under
# /opt/docker/stacks INSIDE the container, but `docker compose up` is executed by the GUEST daemon
# (shared socket), which resolves every relative bind source on the GUEST filesystem. Without a
# SAME-PATH host bind for /opt/docker/stacks, the daemon can't see those bind sources and silently
# creates empty dirs → every bind-mounted stack (base infra AND customer apps) breaks. A named volume
# would NOT fix this (it resolves to /var/lib/docker/volumes/...). Pre-create the dir, then same-path bind.
mkdir -p /opt/docker/stacks
# slice 10 P2: the controller must SEE enrolled external data drives, which the agent binds into the
# guest at /mnt/<name>. Make /mnt a SHARED mount so those binds (and later host-remount self-heal)
# propagate into the controller container, then bind /mnt :rslave (host->container propagation ONLY —
# the container can't mutate the guest mount tree). This is the ONE bind the 8C de-privileging left
# out; scoped to /mnt, which (Model A) holds only Felhom's felhom-data-namespace mounts, never the
# customer's other on-drive data. rbind preserves existing submounts; re-running re-shares (safe).
mkdir -p /mnt
mountpoint -q /mnt || mount --rbind /mnt /mnt
mount --make-rshared /mnt
# Otherwise still DE-PRIVILEGED: disk EXECUTION (scan/format/mount) stays the agent's — NO --privileged,
# no /dev, no /etc/fstab. Bootstrap config (ro), data volume, stacks dir (same-path), the /mnt :rslave
# view, and the docker socket. The controller reaches the agent's local API for disk management.
docker run -d --name felhom-controller --restart unless-stopped "${HOSTNAME_ARGS[@]}" \
  -e FELHOM_BOOTSTRAP_PATH=/etc/felhom-bootstrap/bootstrap.json \
  -v /etc/felhom-bootstrap:/etc/felhom-bootstrap:ro \
  -v felhom-controller-data:/opt/docker/felhom-controller \
  -v /opt/docker/stacks:/opt/docker/stacks \
  -v /mnt:/mnt:rslave \
  -v /var/run/docker.sock:/var/run/docker.sock \
  "$IMAGE"
echo "[ctrl-bootstrap] controller started"
BOOTSH
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.service <<UNIT
[Unit]
Description=Felhom controller bootstrap (deploy the baked controller from the agent-populated config mount)
After=docker.service network-online.target
Wants=docker.service network-online.target
ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/felhom-controller-bootstrap.sh

[Install]
WantedBy=multi-user.target
UNIT
systemctl enable felhom-controller-bootstrap.service'

echo "[golden] baking the first-boot SSH host-key regeneration unit (F3) …"
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-regen-hostkeys.service <<UNIT
[Unit]
Description=Regenerate SSH host keys on first boot if absent
ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
DefaultDependencies=no
After=local-fs.target
Before=ssh.service sshd.service

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/bin/ssh-keygen -A

[Install]
WantedBy=multi-user.target
UNIT
systemctl enable felhom-regen-hostkeys.service'

echo "[golden] identity-clean + minimize …"
pct exec "$VMID" -- bash -c '
  systemctl stop docker containerd 2>/dev/null || true
  apt-get clean; rm -rf /var/lib/apt/lists/*
  rm -f /etc/ssh/ssh_host_*            # regenerated on first boot by the baked unit (F3)
  truncate -s 0 /etc/machine-id        # systemd regenerates on first boot (free)
  rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id
  rm -rf /var/log/*; : > /root/.bash_history
  rm -f /etc/hostname                  # set per-guest at provision (host-side token config)
'

echo "[golden] stop + archive …"
pct stop "$VMID"
# --mode stop with mp0 + mp1 backup=1 → BOTH the Docker-data volume (baked images) and the
# /mnt/sys_drive user-data volume are INCLUDED. The log below MUST show "including mount point mp0"
# AND "including mount point mp1" — if either shows "excluding … (disabled)" the backup flag was lost
# and the archive carries no images / no user-data volume (storage-split B3 trap).
vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true
if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then
  echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting."
  exit 1
fi
if grep -q "excluding volume mount point mp1" /tmp/golden-vzdump.log; then
  echo "[golden] FATAL: mp1 (/mnt/sys_drive) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no user-data volume. Aborting."
  exit 1
fi
grep -q "including mount point mp0" /tmp/golden-vzdump.log \
  || echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive."
grep -q "including mount point mp1" /tmp/golden-vzdump.log \
  || echo "[golden] WARN: could not confirm mp1 inclusion in the vzdump log — verify manually before using this archive."

VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G + user-data ${GOLDEN_SYSDATA_GB}G, all in the archive)"

#-------------------------------------------------------------------------------
# Publish to Gitea (BUNDLE slice) — make this golden fetchable by the host-bootstrap script.
#-------------------------------------------------------------------------------
# The host-install script fetches the golden from Gitea
# (/api/packages/admin/generic/felhom-golden/<golden-version>/golden.tar.zst) and verifies its sha256
# against the hub-vouched artifact manifest before importing it. <golden-version> = the BAKED
# controller version (so the golden's published version tracks what it ships). Publishing is OPT-IN:
# only runs when GITEA_USER + GITEA_TOKEN (or REGISTRY_USER/REGISTRY_TOKEN) are set. The local-golden
# auto-discovery in the host-install script stays as a fallback either way.
GITEA_BASE="${GITEA_BASE:-https://gitea.dooplex.hu}"
GITEA_OWNER="${GITEA_OWNER:-admin}"
PUB_USER="${GITEA_USER:-${REGISTRY_USER:-}}"
PUB_TOKEN="${GITEA_TOKEN:-${REGISTRY_TOKEN:-}}"
# Golden version = the baked controller tag's version (strip the image path + any leading 'v').
GOLDEN_VERSION="${GOLDEN_VERSION:-${CONTROLLER_IMAGE##*:}}"; GOLDEN_VERSION="${GOLDEN_VERSION#v}"

if [ -z "$VOLID" ]; then
  echo "[golden] WARN: could not resolve the archive volid — skipping Gitea publish."
elif [ -z "$PUB_USER" ] || [ -z "$PUB_TOKEN" ]; then
  echo "[golden] Gitea publish SKIPPED (set GITEA_USER+GITEA_TOKEN or REGISTRY_USER+REGISTRY_TOKEN to enable)."
  echo "[golden]   would publish version=$GOLDEN_VERSION from volid $VOLID"
else
  # Resolve the archive's on-disk path (pvesm path turns a volid into a filesystem path).
  ARCHIVE_PATH="$(pvesm path "$VOLID" 2>/dev/null || true)"
  if [ -z "$ARCHIVE_PATH" ] || [ ! -f "$ARCHIVE_PATH" ]; then
    echo "[golden] WARN: cannot resolve archive path for $VOLID — skipping publish."
  else
    GOLDEN_SHA256="$(sha256sum "$ARCHIVE_PATH" | awk '{print $1}')"
    PUB_URL="${GITEA_BASE}/api/packages/${GITEA_OWNER}/generic/felhom-golden/${GOLDEN_VERSION}/golden.tar.zst"
    echo "[golden] publishing golden ($(wc -c < "$ARCHIVE_PATH") bytes, sha256 ${GOLDEN_SHA256:0:16}…) → $PUB_URL"
    # Delete-then-PUT so re-publishing the same version overwrites cleanly (idempotent).
    dcode="$(curl -fsS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X DELETE "$PUB_URL" 2>/dev/null || true)"
    echo "[golden]   pre-delete existing: HTTP ${dcode} (404/204 expected)"
    ucode="$(curl -sS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X PUT --upload-file "$ARCHIVE_PATH" "$PUB_URL")"
    if [ "$ucode" = "201" ] || [ "$ucode" = "200" ]; then
      echo "[golden]   upload OK (HTTP $ucode)"
      echo "GOLDEN_VERSION=${GOLDEN_VERSION}"
      echo "GOLDEN_SHA256=${GOLDEN_SHA256}"
      echo "[golden] Record in the hub operator UI (Configs → Day-0 artifacts): golden ${GOLDEN_VERSION} / ${GOLDEN_SHA256}"
    else
      echo "[golden] WARN: golden upload failed (HTTP $ucode) — the local archive is still usable via auto-discovery."
    fi
  fi
fi

echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"