v0.29.0: OS/Docker-data storage split — golden + provision (Phase 1)

build-golden.sh bakes a small OS rootfs + a dedicated /var/lib/docker volume
(mp0, backup=1) carrying the baked images, plus Docker log rotation. bringup.go
grows the golden's data volume to the per-customer target (DataVolGrowGB) and
emits backup=1 on data mounts (GuestMount.Backup) — closing the spike-B3 silent
DB-loss trap. CLI gains -rootfs-grow/-datavol-grow/-datavol-mount. New
RUNBOOK-provisioning-storage.md. Phase 2 = felhom-controller v0.58.0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

configs/build-golden.sh

@@ -23,6 +23,16 @@
 # Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] [CONTROLLER_IMAGE]
 # Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
 # out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
+#
+# OS / Docker-data SPLIT (storage-split slice): the golden is built with a SMALL OS rootfs and a
+# SEPARATE Docker-data volume mounted at /var/lib/docker (mp0, backup=1). The baked controller +
+# infra images land on that volume and travel INSIDE the golden archive — so provisioned guests boot
+# from baked images with no registry pull. The split is for RESILIENCE: an isolated OS rootfs stays
+# bootable + agent-recoverable if the Docker volume fills (the controller's prevention layer keeps it
+# from filling). Sizes are env-overridable (OS_SIZE_GB / GOLDEN_DOCKER_GB); provision GROWS the data
+# volume to the per-customer target (bringup.go DataVolGrowGB). backup=1 is MANDATORY on the data mp:
+# without it vzdump EXCLUDES the volume (extra LXC mountpoints default backup=0 — storage-split B3),
+# so the archive would carry NO images and provisioned guests would boot imageless.
 set -euo pipefail
 
 VMID="${1:-9100}"
@@ -32,12 +42,17 @@ ARCHIVE_STORAGE="${4:-local}"
 BRIDGE="${5:-vmbr0}"
 CONTROLLER_IMAGE="${6:-gitea.dooplex.hu/admin/felhom-controller:0.43.0}"
 REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
+# OS rootfs size (GiB) and the golden's Docker-data volume size (GiB). Keep GOLDEN_DOCKER_GB just
+# large enough for the baked images + headroom; provision grows it to the per-customer target.
+OS_SIZE_GB="${OS_SIZE_GB:-32}"
+GOLDEN_DOCKER_GB="${GOLDEN_DOCKER_GB:-16}"
 
-echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged) …"
+echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G @ /var/lib/docker, backup=1) …"
 pct create "$VMID" "$TEMPLATE" \
   --hostname felhom-golden --unprivileged 1 \
   --features nesting=1,keyctl=1 \
-  --rootfs "${ROOTFS_STORAGE}:8" --cores 2 --memory 2048 \
+  --rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \
+  --mp0 "${ROOTFS_STORAGE}:${GOLDEN_DOCKER_GB},mp=/var/lib/docker,backup=1" \
   --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0
 
 echo "[golden] starting + installing Docker (official repo, trixie channel) …"
@@ -59,8 +74,20 @@ pct exec "$VMID" -- bash -c '
   apt-get update -qq
   apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
 '
-echo "[golden] verifying Docker works in the build guest …"
-pct exec "$VMID" -- bash -c 'systemctl start docker; sleep 2; docker run --rm hello-world >/dev/null && echo "  docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"))"'
+echo "[golden] baking Docker log rotation into daemon.json (prevention layer: kills unbounded container logs for every guest) …"
+# /var/lib/docker is the mp0 mount (mounted empty before docker installs), so data-root needs NO
+# override — the existing image pulls + this config land on the volume automatically. Only the
+# json-file log caps are set (the most common runaway). Every container inherits these defaults.
+pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json <<JSON
+{
+  "log-driver": "json-file",
+  "log-opts": { "max-size": "10m", "max-file": "3" }
+}
+JSON'
+echo "[golden] verifying Docker works in the build guest (storage driver should be overlayfs on the ext4 data volume) …"
+pct exec "$VMID" -- bash -c 'systemctl start docker; sleep 2; docker run --rm hello-world >/dev/null && echo "  docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); data-root $(docker info 2>/dev/null | sed -n "s/.*Docker Root Dir: //p"))"'
+# Confirm /var/lib/docker is genuinely the dedicated volume, not the rootfs (catch a silent mp miss).
+pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo "  /var/lib/docker is a separate mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker)" || { echo "[golden] FATAL: /var/lib/docker is NOT a separate mount — the mp0 split did not take"; exit 1; }'
 
 echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
 # docker login is used ONCE here on the trusted build host, then logged out before archiving so
@@ -197,8 +224,17 @@ pct exec "$VMID" -- bash -c '
 
 echo "[golden] stop + archive …"
 pct stop "$VMID"
-vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd
+# --mode stop with mp0 backup=1 → the Docker-data volume (with baked images) is INCLUDED. The log
+# below MUST show "including mount point mp0" — if it shows "excluding … (disabled)" the backup flag
+# was lost and the archive carries no images (storage-split B3 trap).
+vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true
+if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then
+  echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting."
+  exit 1
+fi
+grep -q "including mount point mp0" /tmp/golden-vzdump.log \
+  || echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive."
 
 VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
-echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>}"
+echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G, both in the archive)"
 echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"