diff --git a/CHANGELOG.md b/CHANGELOG.md index ebfe93b..a9b0b07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,66 @@ +## v0.120.0 — one data volume (2026-08-03, R-165 · decision D-a · variant V-c) — `build-golden.sh` 2.1.0 → **3.0.0** + +**The dedicated backup partition stops existing.** A golden built by `build-golden.sh` v3.0.0 ships a +**single** data volume; `mp1` is gone. An app whose local recovery unit outgrew the old 20 G area +stopped being backed up even with free space next door — D-a removed that wall rather than moving it. +**A layout change is a major**, hence the script's 3.0.0. + +### The shape: V-c, a neutral mount — chosen on measurement, not on reading + +``` +mp0 -> /var/lib/felhom (one volume, backup=1) + ├─ docker/ --bind--> /var/lib/docker + └─ sys_drive/ --bind--> /mnt/sys_drive +``` + +Three candidates were **built and rebooted**, not reasoned about +(`felhom.eu/documentation/audits/SPIKE-r165-phase0-2026-08-03.md`). All three boot, reboot 3/3, give +ONE `df` figure, keep both paths as real mountpoints, and keep a container's `statfs("/")` reporting +the merged volume. **The ordering worry that motivated the probe did not materialise** — an +`/etc/fstab` bind is ordered under `local-fs.target`, which precedes `docker.service`. + +They differ only in which documented guarantee they break, and that is what decided it: + +- volume at `/var/lib/docker` → customer backups sit **inside Docker's data-root**; the ordinary + "clear `/var/lib/docker` to fix Docker" reflex would destroy every local recovery unit on the box; +- volume at `/mnt/sys_drive` → Docker's **entire data-root** lands under `/mnt`, which the controller + container mounts wholesale (`-v /mnt:/mnt:rslave`). **Measured: the container then sees + `/mnt/sys_drive/docker`**, making the bootstrap's own claim that `/mnt` holds only Felhom's + namespace mounts false. +- **V-c breaks neither**, for one extra path and one extra fstab line. + +### The four assertions were RETARGETED, never deleted — and each was proven to fire + +They exist because a volume silently falling out of the archive is invisible until a restore, which is +the worst moment to learn it. All were re-pointed at the merged invariant and then **run against a +deliberately wrong shape** (a real split-layout guest and a real archive of it): **8 checks, 8 passed.** + +| # | now asserts | proven to reject | +|---|---|---| +| 1 | `/var/lib/docker` is a real mount (the V-c bind) | a plain directory | +| 2 | `/mnt/sys_drive` is a real mount | a plain directory | +| **2b (new)** | both paths are **ONE** filesystem | two volumes — the S2 shape the spike ranked *worse* than the split | +| 4 | the archive's `mp0` is mounted at `/var/lib/felhom` | an `mp0` at the wrong path | +| 5 | the archive carries **no** `mp1` | a leftover `mp1` (the merge-did-not-take case) | + +Assertion 5 replaces the old "was `mp1` excluded?" guard, whose pattern could no longer match after the +merge — **a guard that cannot match is a guard that has silently stopped guarding.** + +### Provisioning: one volume, one grow — and the retired knob is FOLDED IN, not dropped + +`SysDataGrowGB` no longer drives a second resize; its GiB are **added to** the single volume's grow. +**A census established why this matters** — `felhom.eu/scripts/felhom-host-install.sh` computes and +passes `-sysdata-grow`, and an installer and an agent do not upgrade in the same instant. Dropping the +value would silently shrink every appliance an older installer builds by the user-data share (42 of +250 GiB on the standard branch). `-sysdata-grow` / `-sysdata-mount` stay **accepted** for the same +reason; removing them would make every install fail on an unknown flag. `DefaultSysDataMount` now +points at nothing on purpose, so a stale caller is loud at review. + +**§8.2 — the single volume's size IS derived from the physical disk, and already was.** The installer's +`step_grows` reads the thin pool's real free space; the merge only collapsed its 80/20 docker-vs-sysdata +split into one total (`226` = the previous `184+42`, so a standard appliance keeps the same capacity — +250 G, no longer split by a wall). An unflagged install does **not** get the golden's 24 G. + ## CI — the gate entry point runs on every push (2026-08-02, R-168) — NO VERSION BUMP **No version bump, no build, no deploy** — this adds a workflow file only. Stated explicitly so the diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index ff13cf4..f246257 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -165,7 +165,7 @@ func main() { showVersion bool ) flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)") - flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-sysdata-grow/-cores/-memory; keeps the guest)") + flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report; `storage` = observe storage (+ -watch); `backup` = one-shot backup of -vmid; `restore-test` = restore→boot→verify→teardown of -archive (or newest backup); `pbs-verify` = trigger a PBS verify + print snapshot records; `bring-up` = restore→reset identity→size→start link-up of -archive into -vmid (needs -mode/-archive/-vmid; optional -cores/-memory cap; tears down unless -keep); `provision` = full slice-8A chain: bring-up provision + mint token + populate bootstrap config mount (needs -archive/-vmid/-customer-id/-hub-password; optional -rootfs-grow/-datavol-grow/-cores/-memory (-sysdata-grow is deprecated: folded into -datavol-grow); keeps the guest)") flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task|backup|bring-up") flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only") flag.StringVar(&archive, "archive", "", "for --selftest=restore-test|bring-up: the backup volid to restore (restore-test: default newest on the local target)") @@ -175,8 +175,13 @@ func main() { flag.IntVar(&rootfsGrow, "rootfs-grow", 0, "for --selftest=bring-up|provision: grow the OS rootfs by this many GiB after restore (0 = keep golden size)") flag.IntVar(&dataVolGrow, "datavol-grow", 0, "for --selftest=bring-up|provision: grow the golden's Docker-data volume (mp0) by this many GiB (0 = keep golden size)") flag.StringVar(&dataVolMount, "datavol-mount", "", "for --selftest=bring-up|provision: the mpN slot of the Docker-data volume to grow (default mp0)") - flag.IntVar(&sysDataGrow, "sysdata-grow", 0, "for --selftest=bring-up|provision: grow the golden's SSD user-data volume (mp1, /mnt/sys_drive) by this many GiB (0 = keep golden size)") - flag.StringVar(&sysDataMount, "sysdata-mount", "", "for --selftest=bring-up|provision: the mpN slot of the user-data volume to grow (default mp1)") + // R-165: the second volume is gone (build-golden.sh v3.0.0 ships ONE). These two flags are kept + // ACCEPTED because felhom-host-install.sh passes -sysdata-grow and an installer and an agent do not + // upgrade in the same instant — removing them would make every install fail on an unknown flag. + // -sysdata-grow is NOT inert: its GiB are folded into the single volume's grow (bringup.go 4b), so + // an old installer still produces the same total capacity. -sysdata-mount selects nothing. + flag.IntVar(&sysDataGrow, "sysdata-grow", 0, "DEPRECATED (R-165): there is one data volume now; this value is ADDED to -datavol-grow rather than growing a second volume. Kept so an older felhom-host-install.sh keeps working") + flag.StringVar(&sysDataMount, "sysdata-mount", "", "DEPRECATED (R-165): ignored — there is no second volume to select") flag.IntVar(&cores, "cores", 0, "for --selftest=bring-up|provision: cap the guest to N CPU cores (0 = keep golden default). Applied in the pre-start config PUT.") flag.IntVar(&memoryMB, "memory", 0, "for --selftest=bring-up|provision: cap the guest RAM to N MiB (0 = keep golden default). Applied pre-start.") flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)") diff --git a/configs/build-golden.sh b/configs/build-golden.sh index 73997fd..7aa9a83 100644 --- a/configs/build-golden.sh +++ b/configs/build-golden.sh @@ -26,20 +26,42 @@ # Build-time registry login for the controller pull (used ONCE inside the build guest, then logged # out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment. # -# OS / Docker-data SPLIT (storage-split slice): the golden is built with a SMALL OS rootfs and a -# SEPARATE Docker-data volume mounted at /var/lib/docker (mp0, backup=1). The baked controller + -# infra images land on that volume and travel INSIDE the golden archive — so provisioned guests boot -# from baked images with no registry pull. The split is for RESILIENCE: an isolated OS rootfs stays -# bootable + agent-recoverable if the Docker volume fills (the controller's prevention layer keeps it -# from filling). Sizes are env-overridable (OS_SIZE_GB / GOLDEN_DOCKER_GB); provision GROWS the data -# volume to the per-customer target (bringup.go DataVolGrowGB). backup=1 is MANDATORY on the data mp: -# without it vzdump EXCLUDES the volume (extra LXC mountpoints default backup=0 — storage-split B3), -# so the archive would carry NO images and provisioned guests would boot imageless. +# OS / DATA SPLIT, and since v3.0.0 ONE DATA VOLUME (R-165, decision D-a + variant V-c). +# +# The golden is built with a SMALL OS rootfs and a SINGLE data volume (mp0, backup=1) mounted at a +# NEUTRAL path, /var/lib/felhom. Both consumer paths are binds of subdirectories of it: +# +# /var/lib/felhom/docker --bind--> /var/lib/docker (Docker's data-root) +# /var/lib/felhom/sys_drive --bind--> /mnt/sys_drive (the controller's system_data_path) +# +# WHAT THIS REPLACED, AND WHY. Until v2.1.0 these were TWO volumes (mp0 16 G at /var/lib/docker, +# mp1 8 G at /mnt/sys_drive, grown separately at provision). The second one was a fixed ceiling: an +# app whose local recovery unit outgrew it stopped being backed up even with free space next door. +# D-a removed the wall rather than moving it — one volume, one free-space figure, no ceiling. +# +# WHY A NEUTRAL MOUNT AND NOT SIMPLY NESTING ONE PATH INSIDE THE OTHER. Both simpler shapes were +# built and measured (SPIKE-r165-phase0-2026-08-03.md); both boot and reboot cleanly, and each breaks +# a different documented guarantee: +# * volume at /var/lib/docker -> customer backups live INSIDE Docker's data-root, so `du` there +# stops meaning what it says and the ordinary "clear /var/lib/docker to fix Docker" reflex +# destroys every local recovery unit on the box; +# * volume at /mnt/sys_drive -> Docker's ENTIRE data-root lands under /mnt, which the controller +# container mounts wholesale (`-v /mnt:/mnt:rslave`). Measured: the container then sees +# /mnt/sys_drive/docker. The bootstrap's own claim that /mnt "holds only Felhom's +# felhom-data-namespace mounts" would become false. +# The neutral mount breaks neither, for one extra path and one extra fstab line. +# +# The split from the OS rootfs is still for RESILIENCE: an isolated rootfs stays bootable + +# agent-recoverable if the data volume fills (the controller's prevention layer, and since +# controller v0.192.0 the capture floor, keep it from filling). Size is env-overridable +# (OS_SIZE_GB / GOLDEN_VOLUME_GB); provision GROWS the one volume (bringup.go DataVolGrowGB). +# backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra LXC mountpoints default +# backup=0 — storage-split B3), so the archive would carry no images AND no user data. set -euo pipefail # Script provenance — logged into every bake transcript next to the baked controller tag, so an # archive can always be traced to the script that produced it. Bump on any behavior change. -GOLDEN_SCRIPT_VERSION="2.1.0" +GOLDEN_SCRIPT_VERSION="3.0.0" VMID="${1:-9100}" TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}" @@ -59,25 +81,28 @@ if [ -z "$CONTROLLER_IMAGE" ]; then exit 1 fi REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}" -# OS rootfs size (GiB) and the golden's Docker-data volume size (GiB). Keep GOLDEN_DOCKER_GB just -# large enough for the baked images + headroom; provision grows it to the per-customer target. +# OS rootfs size (GiB) and the golden's SINGLE data volume size (GiB). +# +# ONE VOLUME MEANS ONE NUMBER (v3.0.0). The retired GOLDEN_SYSDATA_GB has no successor: there is +# nothing left to size separately. Keep GOLDEN_VOLUME_GB just large enough for the baked images plus +# headroom for the controller's felhom-data skeleton; provision grows the one volume to the +# per-customer target (bringup.go DataVolGrowGB). OS_SIZE_GB="${OS_SIZE_GB:-32}" -GOLDEN_DOCKER_GB="${GOLDEN_DOCKER_GB:-16}" -# The golden's SSD user-data volume (GiB) mounted at /mnt/sys_drive (mp1, backup=1) — the controller's -# system_data_path. Ships small + near-empty (the controller creates /felhom-data itself once -# it's a real mountpoint); provision GROWS it to the per-customer target (bringup.go SysDataGrowGB). Like -# mp0, backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra mountpoints default backup=0 — -# storage-split B3) and the user-data area would silently fall out of PBS coverage. -GOLDEN_SYSDATA_GB="${GOLDEN_SYSDATA_GB:-8}" +# 24 = the retired pair's 16 (docker) + 8 (user-data), so a golden archive carries the same content it +# did before the merge. It is deliberately NOT a per-customer size: provision grows it. +GOLDEN_VOLUME_GB="${GOLDEN_VOLUME_GB:-24}" +# The neutral mount path of the single volume. Both consumer paths are binds of subdirectories of it. +GOLDEN_VOLUME_MP="/var/lib/felhom" echo "[golden] build-golden.sh v${GOLDEN_SCRIPT_VERSION} — baking controller ${CONTROLLER_IMAGE}" -echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G @ /var/lib/docker + user-data ${GOLDEN_SYSDATA_GB}G @ /mnt/sys_drive, both backup=1) …" +echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, backup=1) …" +# ONE mpN slot. There is deliberately no mp1: that slot held the retired user-data volume, and the +# whole point of R-165 is that it stops existing rather than being made bigger. pct create "$VMID" "$TEMPLATE" \ --hostname felhom-golden --unprivileged 1 \ --features nesting=1,keyctl=1 \ --rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \ - --mp0 "${ROOTFS_STORAGE}:${GOLDEN_DOCKER_GB},mp=/var/lib/docker,backup=1" \ - --mp1 "${ROOTFS_STORAGE}:${GOLDEN_SYSDATA_GB},mp=/mnt/sys_drive,backup=1" \ + --mp0 "${ROOTFS_STORAGE}:${GOLDEN_VOLUME_GB},mp=${GOLDEN_VOLUME_MP},backup=1" \ --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0 echo "[golden] starting + installing Docker (official repo, trixie channel) …" @@ -106,8 +131,10 @@ echo "[golden] baking daemon.json: classic overlay2 driver (containerd-snapshott # The classic overlay2 driver stores EVERYTHING (images + overlay + volumes) under data-root # (/var/lib/docker) = the data volume, which is exactly what "one data-root = one partition for all # images + overlay" requires. It also makes the controller's statfs("/") (its overlay root) report the -# DATA volume, which the prevention layer depends on. /var/lib/docker is the mp0 mount (mounted empty -# before docker installs), so data-root needs no override. Log caps kill the most common runaway. +# DATA volume, which the prevention layer depends on — MEASURED to still hold under the v3.0.0 merged +# layout (a container's `df /` reports the single volume, phase-0 spike). Since v3.0.0 /var/lib/docker +# is a BIND of /docker rather than the mp0 mount itself, wired immediately below; data-root +# still needs no override because the path is unchanged. Log caps kill the most common runaway. pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json < /etc/docker/daemon.json "log-opts": { "max-size": "10m", "max-file": "3" } } JSON' +echo "[golden] wiring the single data volume (R-165 variant V-c): ${GOLDEN_VOLUME_MP}/{docker,sys_drive} -> binds …" +# docker-ce has already populated /var/lib/docker ON THE ROOTFS by now (it auto-starts on install), so +# the content is MOVED onto the volume before the bind is laid over the top. Doing it the other way +# round would hide those files under the bind and silently ship a golden whose baked images are on the +# rootfs — the exact failure class the assertions below exist to catch. +# +# /etc/fstab, not a hand-run mount: systemd's fstab generator orders both binds under local-fs.target, +# which precedes basic.target and therefore docker.service. MEASURED across 3 reboots per variant in +# the phase-0 spike — the ordering worry that motivated the probe did not materialise. +pct exec "$VMID" -- bash -c " + set -e + systemctl stop docker docker.socket containerd 2>/dev/null || true + mkdir -p '${GOLDEN_VOLUME_MP}/docker' '${GOLDEN_VOLUME_MP}/sys_drive' + if [ -d /var/lib/docker ] && [ -n \"\$(ls -A /var/lib/docker 2>/dev/null)\" ]; then + cp -a /var/lib/docker/. '${GOLDEN_VOLUME_MP}/docker'/ + rm -rf /var/lib/docker/* + fi + mkdir -p /var/lib/docker /mnt/sys_drive + printf '%s /var/lib/docker none bind 0 0\n' '${GOLDEN_VOLUME_MP}/docker' >> /etc/fstab + printf '%s /mnt/sys_drive none bind 0 0\n' '${GOLDEN_VOLUME_MP}/sys_drive' >> /etc/fstab + systemctl daemon-reload + mount /var/lib/docker + mount /mnt/sys_drive + systemctl start containerd +" + echo "[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) …" # RESTART (not start): docker-ce auto-starts on install with the DEFAULT config, so it is already # running by now; only a restart picks up the daemon.json just written (overlay2 + log caps). @@ -122,12 +175,20 @@ pct exec "$VMID" -- bash -c 'systemctl restart docker; sleep 3; docker run --rm # Guard: the image store MUST be on the data volume now. /var/lib/containerd holding the images would # mean containerd-snapshotter is still on (the split would leave images on the rootfs). pct exec "$VMID" -- bash -c 'drv=$(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); [ "$drv" = "overlay2" ] || { echo "[golden] FATAL: storage driver is $drv, expected overlay2 — images would not land on the data volume"; exit 1; }' -# Confirm /var/lib/docker is genuinely the dedicated volume, not the rootfs (catch a silent mp miss). -pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a separate mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker)" || { echo "[golden] FATAL: /var/lib/docker is NOT a separate mount — the mp0 split did not take"; exit 1; }' -# Same guard for the SSD user-data volume (mp1): /mnt/sys_drive must be its own mount, not the rootfs -# device — otherwise the controller's system_data_path lands on the OS drive and it warns (the whole -# point of this volume is to clear that warning). -pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a separate mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a separate mount — the mp1 split did not take"; exit 1; }' +# ASSERTION 1 (RETARGETED v3.0.0, not removed). /var/lib/docker must be a real mount — now the V-c +# bind of /docker rather than the mp0 mount itself. Still fails closed on the same failure: +# if the bind did not take, Docker's data-root silently sits on the OS rootfs and the golden ships +# its baked images there. +pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a real mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker | head -1)" || { echo "[golden] FATAL: /var/lib/docker is NOT a mount — the V-c docker bind did not take, so the baked images would land on the OS rootfs"; exit 1; }' +# ASSERTION 2 (RETARGETED v3.0.0). /mnt/sys_drive must be a real mount — now the V-c bind of +# /sys_drive. Otherwise the controller's system_data_path lands on the OS drive and it warns +# (clearing that warning is the whole point of the volume). +pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a real mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive | head -1)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a mount — the V-c sys_drive bind did not take, so the controller system_data_path would be the OS rootfs"; exit 1; }' +# ASSERTION 2b (NEW v3.0.0 — the invariant the merge is FOR). Both paths must be backed by the SAME +# device, i.e. ONE filesystem with ONE free-space figure. Two devices here is the S2 shape the R-165 +# spike ranked strictly WORSE than the split it replaced: every assertion satisfied, the ceiling still +# there, and a shared pool neither `df` can see coming. +pct exec "$VMID" -- bash -c 'n=$(df --output=source /var/lib/docker /mnt/sys_drive | tail -n +2 | sort -u | wc -l); [ "$n" = "1" ] && echo " both paths are ONE filesystem: $(df --output=source,avail /var/lib/docker | tail -1)" || { echo "[golden] FATAL: /var/lib/docker and /mnt/sys_drive are on $n DIFFERENT filesystems — that is the S2 shape (two ceilings), not the R-165 merge"; exit 1; }' echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …" # docker login is used ONCE here on the trusted build host, then logged out before archiving so @@ -307,26 +368,36 @@ pct exec "$VMID" -- bash -c ' echo "[golden] stop + archive …" pct stop "$VMID" -# --mode stop with mp0 + mp1 backup=1 → BOTH the Docker-data volume (baked images) and the -# /mnt/sys_drive user-data volume are INCLUDED. The log below MUST show "including mount point mp0" -# AND "including mount point mp1" — if either shows "excluding … (disabled)" the backup flag was lost -# and the archive carries no images / no user-data volume (storage-split B3 trap). +# --mode stop with mp0 backup=1 → the SINGLE data volume (baked images AND the user-data area) is +# INCLUDED. The log MUST show "including mount point mp0" and must NOT show it being excluded — an +# exclusion means the backup flag was lost and the archive carries neither (storage-split B3 trap). +# Since v3.0.0 there is no mp1; the guard that covered it is retargeted below rather than deleted, +# because a guard whose pattern can no longer match is a guard that has silently stopped guarding. vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting." exit 1 fi -if grep -q "excluding volume mount point mp1" /tmp/golden-vzdump.log; then - echo "[golden] FATAL: mp1 (/mnt/sys_drive) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no user-data volume. Aborting." +# ASSERTION 4 (RETARGETED v3.0.0). The mp1 guard used to catch "the user-data volume fell out of the +# archive". After the merge there is no mp1 — so the same failure now looks like the volume being +# mounted at the WRONG PATH, which would carry the images but not the user-data area. Assert the +# inclusion line names the volume's actual mount path. +if ! grep -q "including mount point mp0 ('${GOLDEN_VOLUME_MP}')" /tmp/golden-vzdump.log; then + echo "[golden] FATAL: the archive's mp0 is not ${GOLDEN_VOLUME_MP} — the single data volume is mounted somewhere unexpected, so the archive would not carry both the baked images and the user-data area. Aborting." + grep -iE "mount point" /tmp/golden-vzdump.log || true + exit 1 +fi +# ASSERTION 5 (RETARGETED v3.0.0). There must be NO mp1 in the archive at all. A leftover second +# volume means the merge did not take and this golden would ship the very ceiling R-165 removed. +if grep -qE "mount point mp1" /tmp/golden-vzdump.log; then + echo "[golden] FATAL: the archive still carries an mp1 — the R-165 merge did not take and this golden would ship a second, ceilinged volume. Aborting." exit 1 fi grep -q "including mount point mp0" /tmp/golden-vzdump.log \ || echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive." -grep -q "including mount point mp1" /tmp/golden-vzdump.log \ - || echo "[golden] WARN: could not confirm mp1 inclusion in the vzdump log — verify manually before using this archive." VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1) -echo "[golden] DONE. golden archive volid: ${VOLID:-} (rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G + user-data ${GOLDEN_SYSDATA_GB}G, all in the archive)" +echo "[golden] DONE. golden archive volid: ${VOLID:-} (rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, all in the archive)" #------------------------------------------------------------------------------- # Publish to Gitea (BUNDLE slice) — make this golden fetchable by the host-bootstrap script. diff --git a/internal/reconcile/bringup.go b/internal/reconcile/bringup.go index 0b253fe..adeac4e 100644 --- a/internal/reconcile/bringup.go +++ b/internal/reconcile/bringup.go @@ -51,10 +51,15 @@ const DefaultDataVolMount = "mp0" // Single source of truth for both restore sites (provision bring-up + restore-test). const DefaultPool = "felhom" -// DefaultSysDataMount is the mpN slot the golden bakes the SSD user-data volume (/mnt/sys_drive) at. -// This is the controller's system_data_path; provision grows it (SysDataGrowGB) like the Docker-data -// volume. mp1 is the natural next bring-up slot (mp8/mp9 are added by the provision back-half). -const DefaultSysDataMount = "mp1" +// DefaultSysDataMount is RETIRED (agent v0.120.0, R-165 / decision D-a). The golden no longer bakes a +// second volume: since build-golden.sh v3.0.0 there is ONE data volume at /var/lib/felhom (mp0) and +// both /var/lib/docker and /mnt/sys_drive are binds of subdirectories of it, so there is no mp1 to +// resize. The constant is kept, and deliberately points at nothing, so that a stale caller fails +// loudly at review rather than silently resizing a slot that does not exist. +// +// SysDataGrowGB itself is NOT removed — see its field comment: the host installer still passes +// `-sysdata-grow`, and its GiB are FOLDED INTO the single volume's grow rather than dropped. +const DefaultSysDataMount = "" // Structural host-bind mountpoints every provisioned guest carries (GL-5; verdict of // SPIKE-dr-bindmount-source-2026-07-07): the permanent drives parent bind (mp8, @@ -183,12 +188,18 @@ type BringUpSpec struct { DataVolGrowGB int // DataVolMount is the mpN slot of the golden's Docker-data volume to grow; "" → DefaultDataVolMount ("mp0"). DataVolMount string - // SysDataGrowGB grows the golden-carried SSD user-data volume (SysDataMount, default mp1, mounted at - // /mnt/sys_drive = the controller's system_data_path) to the per-customer target. Same online, - // grow-only mechanism as DataVolGrowGB. 0 = skip (keep the golden's small size — the volume is still - // a separate mount, so the controller's "not a separate drive" warning clears regardless of grow). + // SysDataGrowGB is a COMPATIBILITY INPUT since agent v0.120.0 (R-165). There is no longer a second + // volume to grow — but `felhom.eu/scripts/felhom-host-install.sh` computes and passes + // `-sysdata-grow` (its step_grows derives both numbers from the thin pool's free space), and an + // installer and an agent do not upgrade in the same instant. + // + // SO ITS GiB ARE FOLDED INTO THE SINGLE VOLUME'S GROW RATHER THAN DROPPED. Dropping them would + // silently shrink every appliance by the user-data share — on the ≥300 GiB branch that is 42 of + // 250 GiB — which is exactly the "a knob that silently does nothing" outcome R-165 was told to + // avoid. Folding keeps total capacity identical whichever installer version runs. SysDataGrowGB int - // SysDataMount is the mpN slot of the golden's user-data volume to grow; "" → DefaultSysDataMount ("mp1"). + // SysDataMount is RETIRED and ignored (see DefaultSysDataMount). Kept so an older caller still + // compiles; it selects nothing. SysDataMount string Mounts []GuestMount // additive mpN mounts (slice 7 may pass empty/test) KeepMAC bool // DR knob: keep the archived MAC (true) unless a source may be live @@ -407,12 +418,20 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR // online (storage-split B4); its OWN call like the rootfs resize. The volume + baked images // came in with the restore, so we grow it rather than attach a fresh one that would shadow // the baked images. - if spec.DataVolGrowGB > 0 { + // + // R-165: ONE volume, therefore ONE grow. `SysDataGrowGB` is FOLDED IN here rather than driving + // a second resize — see its field comment. This is the only arithmetic the merge added. + growGB := spec.DataVolGrowGB + spec.SysDataGrowGB + if growGB > 0 { mount := spec.DataVolMount if mount == "" { mount = DefaultDataVolMount } - dupid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.DataVolGrowGB)) + if spec.SysDataGrowGB > 0 { + e.logger.Info("bring-up: folding the retired sys-data grow into the single data volume (R-165)", + "data_grow_gb", spec.DataVolGrowGB, "sysdata_grow_gb", spec.SysDataGrowGB, "total_gb", growGB, "mount", mount) + } + dupid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", growGB)) if err != nil { res.Err = fmt.Errorf("reconcile: bring-up data-volume resize (%s): %w", mount, err) return @@ -423,25 +442,9 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR } } - // 4c. Grow the golden-carried SSD user-data volume (mp1, /mnt/sys_drive = the controller's - // system_data_path) to the per-customer target. Same shape as the Docker-data grow: grow-only, - // online, its OWN call. The volume came in with the restore (separate mount, backup=1), so we - // grow it rather than attach a fresh one. - if spec.SysDataGrowGB > 0 { - mount := spec.SysDataMount - if mount == "" { - mount = DefaultSysDataMount - } - supid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.SysDataGrowGB)) - if err != nil { - res.Err = fmt.Errorf("reconcile: bring-up sys-data resize (%s): %w", mount, err) - return - } - if _, err := e.waitTask(ctx, supid, proxmox.WaitOptions{}); err != nil { - res.Err = fmt.Errorf("reconcile: bring-up sys-data resize task (%s): %w", mount, err) - return - } - } + // 4c. RETIRED (R-165). There is no second volume: the golden ships ONE, and the sys-data grow is + // folded into 4b above. Deliberately left as a comment rather than silently vanishing, so a + // reader of a v0.119.0 archive's provision log can see where the second resize went. // 4d. DR structural-bind swap (GL-5): replace the two restore-time throwaway volumes (see the // restore call) with the REAL host binds, then delete the displaced volumes so a KEPT DR diff --git a/internal/reconcile/bringup_test.go b/internal/reconcile/bringup_test.go index 537d26b..556781d 100644 --- a/internal/reconcile/bringup_test.go +++ b/internal/reconcile/bringup_test.go @@ -265,10 +265,16 @@ func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) { } } -// The golden-carried SSD user-data volume (/mnt/sys_drive) is grown via a SEPARATE resize on its -// mpN slot (mp1), independent of the rootfs and Docker-data grows. With SysDataGrowGB=0 NO mp1 -// resize is issued (the volume stays at the golden size, still a separate mount). -func TestRunBringUp_StorageSplit_SysDataGrow(t *testing.T) { +// R-165 RETARGETED THIS TEST, and the retarget IS the contract change. There is no longer a second +// volume, so `SysDataGrowGB` no longer drives its own resize on mp1 — its GiB are FOLDED INTO the +// single volume's grow. +// +// FOLDED, NOT DROPPED, and that is the whole point. `felhom-host-install.sh` computes and passes +// `-sysdata-grow` from the thin pool's free space, and an installer and an agent do not upgrade in +// the same instant. Dropping the value would silently shrink every appliance built by an older +// installer by the user-data share — 42 of 250 GiB on the standard branch — which is precisely the +// "a knob that silently does nothing" outcome this work was told to avoid. +func TestRunBringUp_StorageSplit_SysDataGrowIsFoldedIn(t *testing.T) { const vmid = 8051 api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}} e, _, q := newEngine(t, api, EmptyProvider{}) @@ -277,26 +283,29 @@ func TestRunBringUp_StorageSplit_SysDataGrow(t *testing.T) { res := e.RunBringUp(context.Background(), BringUpSpec{ Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid, RestoreStorage: "local-lvm", Hostname: "felhom-prov-8051", - DataVolGrowGB: 240, SysDataGrowGB: 42, // grows mp0 AND mp1 (DefaultSysDataMount) + DataVolGrowGB: 240, SysDataGrowGB: 42, // ONE volume: 240 + 42 = 282 }) if res.Err != nil || !res.Pass { t.Fatalf("provision must pass, got %+v", res) } - // TWO resizes here: Docker-data mp0 +240G and the user-data volume mp1 +42G (no rootfs grow). - if len(api.resizes) != 2 { - t.Fatalf("expected data-volume + sys-data resizes, got %+v", api.resizes) + // EXACTLY ONE resize. A second one would mean an mp1 the golden no longer ships. + if len(api.resizes) != 1 { + t.Fatalf("expected exactly ONE data-volume resize (there is no mp1 since R-165), got %+v", api.resizes) } - var sawData, sawSys bool - for _, r := range api.resizes { - if r.disk == "mp0" && r.size == "+240G" { - sawData = true - } - if r.disk == "mp1" && r.size == "+42G" { - sawSys = true - } + r := api.resizes[0] + if r.disk != "mp0" { + t.Fatalf("resized %q, want mp0 — the single data volume", r.disk) } - if !sawData || !sawSys { - t.Errorf("want mp0 +240G AND mp1 +42G, got %+v", api.resizes) + if r.size != "+282G" { + t.Fatalf("resized %s, want +282G (240 data + 42 folded sys-data). Anything less means the "+ + "retired knob's GiB were DROPPED, silently shrinking every appliance an older "+ + "felhom-host-install.sh provisions", r.size) + } + for _, rr := range api.resizes { + if rr.disk == "mp1" { + t.Fatalf("an mp1 resize was issued (%+v) — the golden ships no second volume, so this "+ + "would fail on a real box", rr) + } } }