admin/felhom-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
78c890e4bb8aaaae71dc69035358e0101bdb9e5f
felhom-agent/configs/build-golden.sh
T

512 lines
34 KiB
Bash
Raw Blame History

#!/usr/bin/env bash
# build-golden.sh — build the Felhom golden base LXC archive (slice 7).
#
# Produces a minimal Debian + Docker, unprivileged, nesting=1,keyctl=1, overlayfs LXC, baked
# identity-clean, and archives it for a token-restore by the bring-up reconcile job
# (internal/reconcile/bringup.go). Run as root@pam on a Proxmox host (the keyctl=1 feature flag
# is root-only — phase3 #1; this is the ONE root step, off the per-customer path).
#
# Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993):
# - F3: removing the SSH host keys does NOT auto-regenerate them on Debian (pct restore runs no
# keygen hook), so a baked, Condition-gated first-boot unit regenerates them — keeping the
# agent's front half host-side-only. The gate (ConditionPathExists=!…) makes it fire on a
# provision (golden, keys absent) and no-op on a DR restore (customer backup, keys present),
# symmetric with machine-id.
# - machine-id: truncated; systemd regenerates it on first boot for free (no unit needed).
#
# Slice 8A — the golden now also BAKES the in-guest controller (decision: image baked at golden
# build on the trusted host, so NO registry credential ever enters a customer guest at deploy) and
# a controller-bootstrap unit that, on boot, deploys the baked image from the agent-populated
# config mount (/etc/felhom-bootstrap/bootstrap.json) — no docker login/pull at deploy. Refreshing
# the golden bumps the controller baseline; controller self-update covers in-between drift.
#
# Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] CONTROLLER_IMAGE
# CONTROLLER_IMAGE is REQUIRED (no default) — pass the released controller tag explicitly,
# e.g. gitea.dooplex.hu/admin/felhom-controller:0.98.3.
# Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
# out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
#
# OS / DATA SPLIT, and since v3.0.0 ONE DATA VOLUME (R-165, decision D-a + variant V-c).
#
# The golden is built with a SMALL OS rootfs and a SINGLE data volume (mp0, backup=1) mounted at a
# NEUTRAL path, /var/lib/felhom. Both consumer paths are binds of subdirectories of it:
#
# /var/lib/felhom/docker --bind--> /var/lib/docker (Docker's data-root)
# /var/lib/felhom/sys_drive --bind--> /mnt/sys_drive (the controller's system_data_path)
#
# WHAT THIS REPLACED, AND WHY. Until v2.1.0 these were TWO volumes (mp0 16 G at /var/lib/docker,
# mp1 8 G at /mnt/sys_drive, grown separately at provision). The second one was a fixed ceiling: an
# app whose local recovery unit outgrew it stopped being backed up even with free space next door.
# D-a removed the wall rather than moving it — one volume, one free-space figure, no ceiling.
#
# WHY A NEUTRAL MOUNT AND NOT SIMPLY NESTING ONE PATH INSIDE THE OTHER. Both simpler shapes were
# built and measured (SPIKE-r165-phase0-2026-08-03.md); both boot and reboot cleanly, and each breaks
# a different documented guarantee:
# * volume at /var/lib/docker -> customer backups live INSIDE Docker's data-root, so `du` there
# stops meaning what it says and the ordinary "clear /var/lib/docker to fix Docker" reflex
# destroys every local recovery unit on the box;
# * volume at /mnt/sys_drive -> Docker's ENTIRE data-root lands under /mnt, which the controller
# container mounts wholesale (`-v /mnt:/mnt:rslave`). Measured: the container then sees
# /mnt/sys_drive/docker. The bootstrap's own claim that /mnt "holds only Felhom's
# felhom-data-namespace mounts" would become false.
# The neutral mount breaks neither, for one extra path and one extra fstab line.
#
# The split from the OS rootfs is still for RESILIENCE: an isolated rootfs stays bootable +
# agent-recoverable if the data volume fills (the controller's prevention layer, and since
# controller v0.192.0 the capture floor, keep it from filling). Size is env-overridable
# (OS_SIZE_GB / GOLDEN_VOLUME_GB); provision GROWS the one volume (bringup.go DataVolGrowGB).
# backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra LXC mountpoints default
# backup=0 — storage-split B3), so the archive would carry no images AND no user data.
set -euo pipefail
# Script provenance — logged into every bake transcript next to the baked controller tag, so an
# archive can always be traced to the script that produced it. Bump on any behavior change.
GOLDEN_SCRIPT_VERSION="3.2.0"
VMID="${1:-9100}"
TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}"
ROOTFS_STORAGE="${3:-local-lvm}"
ARCHIVE_STORAGE="${4:-local}"
BRIDGE="${5:-vmbr0}"
# CONTROLLER_IMAGE is MANDATORY — no default. The hand-bumped default rotted twice (0.43.0 →
# 0.85.1 → stale again): each time, a fresh provision would have booted a pre-floor controller
# needing a manual install-day update (drill finding B5). A required argument cannot rot, and
# auto-resolving "latest" could bake a tag the hub manifest never vouched — so the caller states
# the released controller tag explicitly at every rebuild.
CONTROLLER_IMAGE="${6:-}"
if [ -z "$CONTROLLER_IMAGE" ]; then
echo "[golden] FATAL: CONTROLLER_IMAGE (argument 6) is required — pass the released controller tag explicitly." >&2
echo "Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] CONTROLLER_IMAGE" >&2
echo " e.g.: build-golden.sh 9100 local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst local-lvm local vmbr0 gitea.dooplex.hu/admin/felhom-controller:0.98.3" >&2
exit 1
fi
REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
# OS rootfs size (GiB) and the golden's SINGLE data volume size (GiB).
#
# ONE VOLUME MEANS ONE NUMBER (v3.0.0). The retired GOLDEN_SYSDATA_GB has no successor: there is
# nothing left to size separately. Keep GOLDEN_VOLUME_GB just large enough for the baked images plus
# headroom for the controller's felhom-data skeleton; provision grows the one volume to the
# per-customer target (bringup.go DataVolGrowGB).
OS_SIZE_GB="${OS_SIZE_GB:-32}"
# 24 = the retired pair's 16 (docker) + 8 (user-data), so a golden archive carries the same content it
# did before the merge. It is deliberately NOT a per-customer size: provision grows it.
GOLDEN_VOLUME_GB="${GOLDEN_VOLUME_GB:-24}"
# The neutral mount path of the single volume. Both consumer paths are binds of subdirectories of it.
GOLDEN_VOLUME_MP="/var/lib/felhom"
echo "[golden] build-golden.sh v${GOLDEN_SCRIPT_VERSION} — baking controller ${CONTROLLER_IMAGE}"
echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, backup=1) …"
# ONE mpN slot. There is deliberately no mp1: that slot held the retired user-data volume, and the
# whole point of R-165 is that it stops existing rather than being made bigger.
pct create "$VMID" "$TEMPLATE" \
--hostname felhom-golden --unprivileged 1 \
--features nesting=1,keyctl=1 \
--rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \
--mp0 "${ROOTFS_STORAGE}:${GOLDEN_VOLUME_GB},mp=${GOLDEN_VOLUME_MP},backup=1" \
--net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0
echo "[golden] starting + installing Docker (official repo, trixie channel) …"
pct start "$VMID"
# wait for DHCP/DNS
for i in $(seq 1 30); do
if pct exec "$VMID" -- getent hosts download.docker.com >/dev/null 2>&1; then break; fi
sleep 1
done
# v3.1.0 (`11` §5.8): GOLDEN_DOCKER_PKGS pins the APPROVED Docker engine set (the hub's newest Docker release, all six
# "name=version"); without it the newest stable set is installed and the bake log says so.
GOLDEN_DOCKER_PKGS="${GOLDEN_DOCKER_PKGS:-}"
if [[ -n "$GOLDEN_DOCKER_PKGS" ]]; then
echo "[golden] Docker engine set PINNED to the approved release: $GOLDEN_DOCKER_PKGS"
else
echo "[golden] WARNING: GOLDEN_DOCKER_PKGS not set — installing the newest stable Docker set, not an approved one"
fi
pct exec "$VMID" -- env GOLDEN_DOCKER_PKGS="$GOLDEN_DOCKER_PKGS" bash -c '
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq ca-certificates curl >/dev/null
install -m0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" \
> /etc/apt/sources.list.d/docker.list
apt-get update -qq
if [ -n "$GOLDEN_DOCKER_PKGS" ]; then
apt-get install -y -qq $GOLDEN_DOCKER_PKGS >/dev/null
else
apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
fi
dpkg-query -W containerd.io docker-buildx-plugin docker-ce docker-ce-cli docker-ce-rootless-extras docker-compose-plugin 2>/dev/null | sed "s/^/ installed: /"
'
# v3.2.0 (Part F of the R-840 brief, `11` §5.3): GOLDEN_GUEST_PKGS = the newest APPROVED guest release, as
# "name=version …" (the hub's os_releases row for layer guest, IN FORCE — never a cancelled test approval). The bake
# brings every package the template HAS to exactly that version, under felhom-os-apply's rules: never a package the
# template lacks (--only-upgrade), never newer than approved, never a removal or a new package (a simulation is checked
# first and the bake FAILS on either). Empty = no approved guest release in force: the template's versions stay, and
# the box's first night installs whatever release is approved then. Either way the bake PRINTS the first-night count:
# how many installed packages are older than the approved version (target 0).
GOLDEN_GUEST_PKGS="${GOLDEN_GUEST_PKGS:-}"
pct exec "$VMID" -- env GOLDEN_GUEST_PKGS="$GOLDEN_GUEST_PKGS" bash -c '
set -e
export DEBIAN_FRONTEND=noninteractive
if [ -z "$GOLDEN_GUEST_PKGS" ]; then
echo "[golden] no approved guest release given - the template versions stay; first-night count vs an approved release: n/a"
echo "[golden] pending Debian upgrades in the baked guest (what a FUTURE approval may bring): $(apt list --upgradable 2>/dev/null | grep -c /)"
exit 0
fi
want=""
for nv in $GOLDEN_GUEST_PKGS; do
n=${nv%%=*}; v=${nv#*=}
cur=$(dpkg-query -W -f="\${Version}" "$n" 2>/dev/null) || continue # not in the template: never added
dpkg --compare-versions "$cur" lt "$v" && want="$want $n=$v"
done
if [ -n "$want" ]; then
sim=$(apt-get -s install --only-upgrade -o Dpkg::Options::=--force-confold $want)
if echo "$sim" | grep -q "^Remv "; then echo "[golden] FATAL: the approved guest set would REMOVE a package"; echo "$sim" | grep "^Remv "; exit 1; fi
for p in $(echo "$sim" | awk "/^Inst /{print \$2}"); do
dpkg-query -W "$p" >/dev/null 2>&1 || { echo "[golden] FATAL: the approved guest set would ADD $p - not in the template"; exit 1; }
done
apt-get install -y -qq --only-upgrade -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef $want >/dev/null
echo "[golden] approved guest release installed: $(echo $want | wc -w) package(s) brought to the approved version"
else
echo "[golden] approved guest release: the template already runs every approved version"
fi
left=0
for nv in $GOLDEN_GUEST_PKGS; do
n=${nv%%=*}; v=${nv#*=}
cur=$(dpkg-query -W -f="\${Version}" "$n" 2>/dev/null) || continue
dpkg --compare-versions "$cur" lt "$v" && { left=$((left+1)); echo " still older: $n $cur < $v"; }
done
echo "[golden] first-night count vs the approved guest release: $left (target 0)"
[ "$left" -eq 0 ] || { echo "[golden] FATAL: $left package(s) stayed older than the approved release"; exit 1; }
'
echo "[golden] baking daemon.json: classic overlay2 driver (containerd-snapshotter OFF) + log rotation …"
# containerd-snapshotter (Docker 28+/29 default) keeps the IMAGE content store under
# /var/lib/containerd — which is NOT /var/lib/docker, so it would stay on the OS rootfs and the split
# would only move named volumes, defeating the whole point (validated: images landed on the rootfs).
# The classic overlay2 driver stores EVERYTHING (images + overlay + volumes) under data-root
# (/var/lib/docker) = the data volume, which is exactly what "one data-root = one partition for all
# images + overlay" requires. It also makes the controller's statfs("/") (its overlay root) report the
# DATA volume, which the prevention layer depends on — MEASURED to still hold under the v3.0.0 merged
# layout (a container's `df /` reports the single volume, phase-0 spike). Since v3.0.0 /var/lib/docker
# is a BIND of <volume>/docker rather than the mp0 mount itself, wired immediately below; data-root
# still needs no override because the path is unchanged. Log caps kill the most common runaway.
# v3.1.0: "live-restore": true (`09` decision 87) — a Docker engine update then restarts no app (`11` C5). A box made
# from this golden never needs the agent's one-time live-restore-on step. NEVER removed by a plain restart (R-835).
pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json <<JSON
{
"features": { "containerd-snapshotter": false },
"live-restore": true,
"log-driver": "json-file",
"log-opts": { "max-size": "10m", "max-file": "3" }
}
JSON'
echo "[golden] wiring the single data volume (R-165 variant V-c): ${GOLDEN_VOLUME_MP}/{docker,sys_drive} -> binds …"
# docker-ce has already populated /var/lib/docker ON THE ROOTFS by now (it auto-starts on install), so
# the content is MOVED onto the volume before the bind is laid over the top. Doing it the other way
# round would hide those files under the bind and silently ship a golden whose baked images are on the
# rootfs — the exact failure class the assertions below exist to catch.
#
# /etc/fstab, not a hand-run mount: systemd's fstab generator orders both binds under local-fs.target,
# which precedes basic.target and therefore docker.service. MEASURED across 3 reboots per variant in
# the phase-0 spike — the ordering worry that motivated the probe did not materialise.
pct exec "$VMID" -- bash -c "
set -e
systemctl stop docker docker.socket containerd 2>/dev/null || true
mkdir -p '${GOLDEN_VOLUME_MP}/docker' '${GOLDEN_VOLUME_MP}/sys_drive'
if [ -d /var/lib/docker ] && [ -n \"\$(ls -A /var/lib/docker 2>/dev/null)\" ]; then
cp -a /var/lib/docker/. '${GOLDEN_VOLUME_MP}/docker'/
rm -rf /var/lib/docker/*
fi
mkdir -p /var/lib/docker /mnt/sys_drive
printf '%s /var/lib/docker none bind 0 0\n' '${GOLDEN_VOLUME_MP}/docker' >> /etc/fstab
printf '%s /mnt/sys_drive none bind 0 0\n' '${GOLDEN_VOLUME_MP}/sys_drive' >> /etc/fstab
systemctl daemon-reload
mount /var/lib/docker
mount /mnt/sys_drive
systemctl start containerd
"
echo "[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) …"
# RESTART (not start): docker-ce auto-starts on install with the DEFAULT config, so it is already
# running by now; only a restart picks up the daemon.json just written (overlay2 + log caps).
pct exec "$VMID" -- bash -c 'systemctl restart docker; sleep 3; docker run --rm hello-world >/dev/null && echo " docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); data-root $(docker info 2>/dev/null | sed -n "s/.*Docker Root Dir: //p"))"'
# Guard: the image store MUST be on the data volume now. /var/lib/containerd holding the images would
# mean containerd-snapshotter is still on (the split would leave images on the rootfs).
pct exec "$VMID" -- bash -c 'drv=$(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); [ "$drv" = "overlay2" ] || { echo "[golden] FATAL: storage driver is $drv, expected overlay2 — images would not land on the data volume"; exit 1; }'
# v3.1.0 ASSERTION: live-restore is ON in the running daemon (decision 87), or the bake fails closed.
pct exec "$VMID" -- bash -c 'lr=$(docker info --format "{{.LiveRestoreEnabled}}" 2>/dev/null); [ "$lr" = "true" ] && echo " live-restore: on" || { echo "[golden] FATAL: live-restore is $lr, expected true (decision 87)"; exit 1; }'
# ASSERTION 1 (RETARGETED v3.0.0, not removed). /var/lib/docker must be a real mount — now the V-c
# bind of <volume>/docker rather than the mp0 mount itself. Still fails closed on the same failure:
# if the bind did not take, Docker's data-root silently sits on the OS rootfs and the golden ships
# its baked images there.
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a real mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker | head -1)" || { echo "[golden] FATAL: /var/lib/docker is NOT a mount — the V-c docker bind did not take, so the baked images would land on the OS rootfs"; exit 1; }'
# ASSERTION 2 (RETARGETED v3.0.0). /mnt/sys_drive must be a real mount — now the V-c bind of
# <volume>/sys_drive. Otherwise the controller's system_data_path lands on the OS drive and it warns
# (clearing that warning is the whole point of the volume).
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a real mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive | head -1)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a mount — the V-c sys_drive bind did not take, so the controller system_data_path would be the OS rootfs"; exit 1; }'
# ASSERTION 2b (NEW v3.0.0 — the invariant the merge is FOR). Both paths must be backed by the SAME
# device, i.e. ONE filesystem with ONE free-space figure. Two devices here is the S2 shape the R-165
# spike ranked strictly WORSE than the split it replaced: every assertion satisfied, the ceiling still
# there, and a shared pool neither `df` can see coming.
pct exec "$VMID" -- bash -c 'n=$(df --output=source /var/lib/docker /mnt/sys_drive | tail -n +2 | sort -u | wc -l); [ "$n" = "1" ] && echo " both paths are ONE filesystem: $(df --output=source,avail /var/lib/docker | tail -1)" || { echo "[golden] FATAL: /var/lib/docker and /mnt/sys_drive are on $n DIFFERENT filesystems — that is the S2 shape (two ceilings), not the R-165 merge"; exit 1; }'
echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
# docker login is used ONCE here on the trusted build host, then logged out before archiving so
# the credential is NEVER baked into the golden. The IMAGE is what gets baked (in Docker storage).
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null"
fi
pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'"
# Record the baked image ref for the bootstrap unit (so the unit needs no login/pull).
pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image"
# Bake EVERY controller-managed infra image so enabling an infra stack on a fresh box is near-instant
# instead of a multi-minute silent registry pull.
#
# THE LIST COMES FROM THE CONTROLLER WE JUST PULLED, not from a copy here. This script used to carry
# a hand-maintained array of three tags with a comment telling the reader to keep it in sync with the
# controller's internal/infra constants. It drifted the moment a fourth stack was added: felhom-samba
# was never added here, so the golden baked 3 of 4 and turning on Megosztás pulled from the registry
# with zero UI feedback — observed live, twice. Asking the binary (`--print-infra-images`, backed by
# infra.Images()) makes golden-vs-controller drift structurally impossible: the golden bakes exactly
# what THIS controller version will request.
echo "[golden] asking the controller which infra images it manages …"
INFRA_LIST="$(pct exec "$VMID" -- bash -c "docker run --rm --entrypoint /usr/local/bin/felhom-controller '$CONTROLLER_IMAGE' --print-infra-images 2>/dev/null" || true)"
mapfile -t INFRA_IMAGES < <(printf '%s\n' "$INFRA_LIST" | grep -E '^[a-z0-9._/-]+:[A-Za-z0-9._-]+$' || true)
if [ "${#INFRA_IMAGES[@]}" -eq 0 ]; then
# Controllers older than v0.147.0 have no --print-infra-images. Fall back to the historical list so
# an old-controller bake still works — but say so loudly, because the fallback is exactly the
# drift-prone thing this change removed, and it CANNOT know about stacks added after it was written.
echo "[golden] WARN: '$CONTROLLER_IMAGE' does not support --print-infra-images (pre-0.147.0)."
echo "[golden] WARN: falling back to the historical 3-image list. felhom-samba will NOT be baked,"
echo "[golden] WARN: so enabling Megosztás on this golden will pull at runtime. Bake >=0.147.0."
INFRA_IMAGES=(
"traefik:v3.6.7"
"cloudflare/cloudflared:2026.6.0"
"gtstef/filebrowser:1.3.3-stable"
)
fi
echo "[golden] baking infra images (${#INFRA_IMAGES[@]}): ${INFRA_IMAGES[*]} …"
for img in "${INFRA_IMAGES[@]}"; do
# Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails
# mid-bake with a confusing error).
pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \
|| { echo "[golden] FATAL: pinned infra image does not resolve: $img"; exit 1; }
pct exec "$VMID" -- bash -c "docker pull '$img'"
done
# Log out LAST — not right after the controller pull. felhom-samba lives on the same private registry
# as the controller, so the infra loop above needs the credential; logging out first made that pull
# 401. The credential is still never archived: the logout + config.json removal happen before the
# guest is stopped and templated.
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
pct exec "$VMID" -- bash -c '[ ! -s /root/.docker/config.json ]' \
|| { echo "[golden] FATAL: registry credential still present in the guest — refusing to archive"; exit 1; }
echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …"
pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH'
#!/bin/bash
# felhom controller-bootstrap (slice 8A): the host agent's back-half populated the read-only
# config mount /etc/felhom-bootstrap; this golden-baked oneshot deploys the BAKED controller image
# with that config. NO docker login / NO docker pull — the image is already in this golden's Docker
# storage (and self-update handles version drift). Host-side only; the agent never enters the guest.
set -euo pipefail
CFG=/etc/felhom-bootstrap/bootstrap.json
[ -r "$CFG" ] || { echo "[ctrl-bootstrap] no $CFG — not provisioned, nothing to do"; exit 0; }
IMAGE=$(cat /etc/felhom-controller-image 2>/dev/null || true)
[ -n "$IMAGE" ] || { echo "[ctrl-bootstrap] FATAL: /etc/felhom-controller-image missing"; exit 1; }
# Per-guest container hostname (slice base-infra/3A): derive from the bootstrap's customer.id so the
# controller's os.Hostname() (its hub-reported hostname) is the customer id, not the Docker container
# ID. Portable, dependency-free parse (NO jq in the golden) — bootstrap.json has exactly one "id" key
# (customer.id). Falls back to no --hostname if the parse yields nothing (fail-safe).
CUSTOMER_ID=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CFG" | head -1)
# SECURITY: $CUSTOMER_ID is interpolated into `docker run` — reject anything that isn't a DNS-safe
# label so a malformed/hostile customer.id can't smuggle extra docker flags (e.g. --privileged, -v).
# Then pass it via a quoted array (never word-split). Empty/invalid → no --hostname (fail-safe).
case "$CUSTOMER_ID" in
""|*[!a-zA-Z0-9._-]*|-*) CUSTOMER_ID="" ;;
esac
HOSTNAME_ARGS=()
[ -n "$CUSTOMER_ID" ] && HOSTNAME_ARGS=(--hostname "$CUSTOMER_ID")
echo "[ctrl-bootstrap] deploying $IMAGE from $CFG (hostname=${CUSTOMER_ID:-<unset>})"
docker rm -f felhom-controller >/dev/null 2>&1 || true
# Section-G fix (base-infra slice): the controller writes app/infra compose stacks under
# /opt/docker/stacks INSIDE the container, but `docker compose up` is executed by the GUEST daemon
# (shared socket), which resolves every relative bind source on the GUEST filesystem. Without a
# SAME-PATH host bind for /opt/docker/stacks, the daemon can't see those bind sources and silently
# creates empty dirs → every bind-mounted stack (base infra AND customer apps) breaks. A named volume
# would NOT fix this (it resolves to /var/lib/docker/volumes/...). Pre-create the dir, then same-path bind.
mkdir -p /opt/docker/stacks
# slice 10 P2: the controller must SEE enrolled external data drives, which the agent binds into the
# guest at /mnt/<name>. Make /mnt a SHARED mount so those binds (and later host-remount self-heal)
# propagate into the controller container, then bind /mnt :rslave (host->container propagation ONLY —
# the container can't mutate the guest mount tree). This is the ONE bind the 8C de-privileging left
# out; scoped to /mnt, which (Model A) holds only Felhom's felhom-data-namespace mounts, never the
# customer's other on-drive data. rbind preserves existing submounts; re-running re-shares (safe).
mkdir -p /mnt
mountpoint -q /mnt || mount --rbind /mnt /mnt
mount --make-rshared /mnt
# Otherwise still DE-PRIVILEGED: disk EXECUTION (scan/format/mount) stays the agent's — NO --privileged,
# no /dev, no /etc/fstab. Bootstrap config (ro), data volume, stacks dir (same-path), the /mnt :rslave
# view, and the docker socket. The controller reaches the agent's local API for disk management.
# R-523: `always`, not `unless-stopped`. It covers ONE extra case only — a Docker daemon restart after
# the container was stopped by hand. Neither policy restarts a container that `docker kill`/`docker
# stop` ended (measured 2026-09-15, Docker 29.8.0, evidence-p1fixes-2026-09-15/A1); the host agent's
# controller supervisor (felhom-agent v0.131.0, internal/localapi/controllersupervisor.go) covers that.
docker run -d --name felhom-controller --restart always "${HOSTNAME_ARGS[@]}" \
-e FELHOM_BOOTSTRAP_PATH=/etc/felhom-bootstrap/bootstrap.json \
-v /etc/felhom-bootstrap:/etc/felhom-bootstrap:ro \
-v felhom-controller-data:/opt/docker/felhom-controller \
-v /opt/docker/stacks:/opt/docker/stacks \
-v /mnt:/mnt:rslave \
-v /var/run/docker.sock:/var/run/docker.sock \
"$IMAGE"
echo "[ctrl-bootstrap] controller started"
BOOTSH
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.service <<UNIT
[Unit]
Description=Felhom controller bootstrap (deploy the baked controller from the agent-populated config mount)
After=docker.service network-online.target
Wants=docker.service network-online.target
ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/felhom-controller-bootstrap.sh
[Install]
WantedBy=multi-user.target
UNIT
systemctl enable felhom-controller-bootstrap.service'
# B1 (DRILL-day0-cleanroom-2026-07-03 R6/B1): the service's ConditionPathExists is evaluated only
# when the service is STARTED — i.e. at boot via multi-user.target — but the agent back-half
# hot-plugs the bootstrap mount into the ALREADY-RUNNING guest, so on a provision the boot-time
# start races the mount and loses on slow hardware. This path unit watches for bootstrap.json and
# starts the service when it APPEARS — covering the provision hot-plug without a reboot. The boot
# case is still served by the enabled service itself; RemainAfterExit=yes on the service means the
# path unit cannot re-trigger it in a loop once it has run.
echo "[golden] baking the controller-bootstrap PATH unit (starts the service on bootstrap-mount hot-plug — B1) …"
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.path <<UNIT
[Unit]
Description=Watch for the agent-populated bootstrap config; start the controller bootstrap when it appears
[Path]
PathExists=/etc/felhom-bootstrap/bootstrap.json
[Install]
WantedBy=multi-user.target
UNIT
systemctl enable felhom-controller-bootstrap.path'
echo "[golden] baking the first-boot SSH host-key regeneration unit (F3) …"
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-regen-hostkeys.service <<UNIT
[Unit]
Description=Regenerate SSH host keys on first boot if absent
ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
DefaultDependencies=no
After=local-fs.target
Before=ssh.service sshd.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/bin/ssh-keygen -A
[Install]
WantedBy=multi-user.target
UNIT
systemctl enable felhom-regen-hostkeys.service'
echo "[golden] identity-clean + minimize …"
pct exec "$VMID" -- bash -c '
systemctl stop docker containerd 2>/dev/null || true
apt-get clean; rm -rf /var/lib/apt/lists/*
rm -f /etc/ssh/ssh_host_* # regenerated on first boot by the baked unit (F3)
truncate -s 0 /etc/machine-id # systemd regenerates on first boot (free)
rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id
rm -rf /var/log/*; : > /root/.bash_history
rm -f /etc/hostname # set per-guest at provision (host-side token config)
'
echo "[golden] stop + archive …"
pct stop "$VMID"
# --mode stop with mp0 backup=1 → the SINGLE data volume (baked images AND the user-data area) is
# INCLUDED. The log MUST show "including mount point mp0" and must NOT show it being excluded — an
# exclusion means the backup flag was lost and the archive carries neither (storage-split B3 trap).
# Since v3.0.0 there is no mp1; the guard that covered it is retargeted below rather than deleted,
# because a guard whose pattern can no longer match is a guard that has silently stopped guarding.
vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true
if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then
echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting."
exit 1
fi
# ASSERTION 4 (RETARGETED v3.0.0). The mp1 guard used to catch "the user-data volume fell out of the
# archive". After the merge there is no mp1 — so the same failure now looks like the volume being
# mounted at the WRONG PATH, which would carry the images but not the user-data area. Assert the
# inclusion line names the volume's actual mount path.
if ! grep -q "including mount point mp0 ('${GOLDEN_VOLUME_MP}')" /tmp/golden-vzdump.log; then
echo "[golden] FATAL: the archive's mp0 is not ${GOLDEN_VOLUME_MP} — the single data volume is mounted somewhere unexpected, so the archive would not carry both the baked images and the user-data area. Aborting."
grep -iE "mount point" /tmp/golden-vzdump.log || true
exit 1
fi
# ASSERTION 5 (RETARGETED v3.0.0). There must be NO mp1 in the archive at all. A leftover second
# volume means the merge did not take and this golden would ship the very ceiling R-165 removed.
if grep -qE "mount point mp1" /tmp/golden-vzdump.log; then
echo "[golden] FATAL: the archive still carries an mp1 — the R-165 merge did not take and this golden would ship a second, ceilinged volume. Aborting."
exit 1
fi
grep -q "including mount point mp0" /tmp/golden-vzdump.log \
|| echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive."
VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, all in the archive)"
#-------------------------------------------------------------------------------
# Publish to Gitea (BUNDLE slice) — make this golden fetchable by the host-bootstrap script.
#-------------------------------------------------------------------------------
# The host-install script fetches the golden from Gitea
# (/api/packages/admin/generic/felhom-golden/<golden-version>/golden.tar.zst) and verifies its sha256
# against the hub-vouched artifact manifest before importing it. <golden-version> = the BAKED
# controller version (so the golden's published version tracks what it ships). Publishing is OPT-IN:
# only runs when GITEA_USER + GITEA_TOKEN (or REGISTRY_USER/REGISTRY_TOKEN) are set. The local-golden
# auto-discovery in the host-install script stays as a fallback either way.
GITEA_BASE="${GITEA_BASE:-https://gitea.dooplex.hu}"
GITEA_OWNER="${GITEA_OWNER:-admin}"
PUB_USER="${GITEA_USER:-${REGISTRY_USER:-}}"
PUB_TOKEN="${GITEA_TOKEN:-${REGISTRY_TOKEN:-}}"
# Golden version = the baked controller tag's version (strip the image path + any leading 'v').
GOLDEN_VERSION="${GOLDEN_VERSION:-${CONTROLLER_IMAGE##*:}}"; GOLDEN_VERSION="${GOLDEN_VERSION#v}"
if [ -z "$VOLID" ]; then
echo "[golden] WARN: could not resolve the archive volid — skipping Gitea publish."
elif [ -z "$PUB_USER" ] || [ -z "$PUB_TOKEN" ]; then
echo "[golden] Gitea publish SKIPPED (set GITEA_USER+GITEA_TOKEN or REGISTRY_USER+REGISTRY_TOKEN to enable)."
echo "[golden] would publish version=$GOLDEN_VERSION from volid $VOLID"
else
# Resolve the archive's on-disk path (pvesm path turns a volid into a filesystem path).
ARCHIVE_PATH="$(pvesm path "$VOLID" 2>/dev/null || true)"
if [ -z "$ARCHIVE_PATH" ] || [ ! -f "$ARCHIVE_PATH" ]; then
echo "[golden] WARN: cannot resolve archive path for $VOLID — skipping publish."
else
GOLDEN_SHA256="$(sha256sum "$ARCHIVE_PATH" | awk '{print $1}')"
PUB_URL="${GITEA_BASE}/api/packages/${GITEA_OWNER}/generic/felhom-golden/${GOLDEN_VERSION}/golden.tar.zst"
echo "[golden] publishing golden ($(wc -c < "$ARCHIVE_PATH") bytes, sha256 ${GOLDEN_SHA256:0:16}…) → $PUB_URL"
# Delete-then-PUT so re-publishing the same version overwrites cleanly (idempotent).
dcode="$(curl -fsS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X DELETE "$PUB_URL" 2>/dev/null || true)"
echo "[golden] pre-delete existing: HTTP ${dcode} (404/204 expected)"
ucode="$(curl -sS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X PUT --upload-file "$ARCHIVE_PATH" "$PUB_URL")"
if [ "$ucode" = "201" ] || [ "$ucode" = "200" ]; then
echo "[golden] upload OK (HTTP $ucode)"
echo "GOLDEN_VERSION=${GOLDEN_VERSION}"
echo "GOLDEN_SHA256=${GOLDEN_SHA256}"
echo "[golden] Record in the hub operator UI (Configs → Day-0 artifacts): golden ${GOLDEN_VERSION} / ${GOLDEN_SHA256}"
else
echo "[golden] WARN: golden upload failed (HTTP $ucode) — the local archive is still usable via auto-discovery."
fi
fi
fi
echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"
Reference in New Issue View Git Blame Copy Permalink