v0.29.0: OS/Docker-data storage split — golden + provision (Phase 1)

build-golden.sh bakes a small OS rootfs + a dedicated /var/lib/docker volume (mp0, backup=1) carrying the baked images, plus Docker log rotation. bringup.go grows the golden's data volume to the per-customer target (DataVolGrowGB) and emits backup=1 on data mounts (GuestMount.Backup) — closing the spike-B3 silent DB-loss trap. CLI gains -rootfs-grow/-datavol-grow/-datavol-mount. New RUNBOOK-provisioning-storage.md. Phase 2 = felhom-controller v0.58.0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-13 15:38:26 +02:00
parent 5ab159521d
commit d7d68fdd83
6 changed files with 275 additions and 11 deletions
@@ -40,6 +40,9 @@ const (

 const bringUpKind = "bring_up"

+// DefaultDataVolMount is the mpN slot the golden bakes the Docker-data volume (/var/lib/docker) at.
+const DefaultDataVolMount = "mp0"
+
 // configLockMaxAttempts bounds the F4 config-lock retry. configLockBackoff is a package var so
 // tests can shrink it (the production value gives PVE time to release its async config lock).
 const configLockMaxAttempts = 5
@@ -52,6 +55,12 @@ type GuestMount struct {
 	Storage    string // PVE storage id (e.g. "local-lvm")
 	SizeGB     int    // new-volume size in GiB
 	MountPoint string // in-guest path (e.g. "/mnt/data")
+	// Backup includes this mountpoint in vzdump/PBS. MANDATORY for any data-bearing mount (DB
+	// volumes), because extra LXC mountpoints default to backup=0 = EXCLUDED from the snapshot
+	// (storage-split finding B3). The Docker-data volume normally rides in from the golden archive
+	// (already backup=1) and is grown via DataVolGrowGB rather than attached here, but any data
+	// mount attached through spec.Mounts MUST set this or its contents silently fall out of PBS.
+	Backup bool
 }

 // BringUpSpec is the input to one bring-up. The caller resolves it (the selftest, or slice-10
@@ -65,7 +74,14 @@ type BringUpSpec struct {
 	Cores          int           // 0 = leave as restored
 	MemoryMB       int           // 0 = leave as restored
 	RootfsGrowGB   int           // optional grow-only rootfs resize (0 = skip)
-	Mounts         []GuestMount  // additive mpN mounts (slice 7 may pass empty/test)
+	// DataVolGrowGB grows the golden-carried Docker-data volume (DataVolMount, default mp0) to the
+	// per-customer target. The golden ships a small data volume with the baked images; provision
+	// grows it online (grow-only, storage-split B4) rather than attaching a fresh empty volume that
+	// would shadow the baked images. 0 = skip (keep the golden's size).
+	DataVolGrowGB int
+	// DataVolMount is the mpN slot of the golden's Docker-data volume to grow; "" → DefaultDataVolMount ("mp0").
+	DataVolMount string
+	Mounts       []GuestMount  // additive mpN mounts (slice 7 may pass empty/test)
 	KeepMAC        bool          // DR knob: keep the archived MAC (true) unless a source may be live
 	BootTimeout    time.Duration // 0 → DefaultBootTimeout; bounds the link-up liveness wait
 }
@@ -213,6 +229,26 @@ func (e *Engine) runBringUp(ctx context.Context, spec BringUpSpec, res *BringUpR
 		}
 	}

+	// 4b. Grow the golden-carried Docker-data volume (mp0) to the per-customer target. Grow-only,
+	//     online (storage-split B4); its OWN call like the rootfs resize. The volume + baked images
+	//     came in with the restore, so we grow it rather than attach a fresh one that would shadow
+	//     the baked images.
+	if spec.DataVolGrowGB > 0 {
+		mount := spec.DataVolMount
+		if mount == "" {
+			mount = DefaultDataVolMount
+		}
+		dupid, err := e.api.ResizeLXC(ctx, spec.VMID, mount, fmt.Sprintf("+%dG", spec.DataVolGrowGB))
+		if err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up data-volume resize (%s): %w", mount, err)
+			return
+		}
+		if _, err := e.waitTask(ctx, dupid, proxmox.WaitOptions{}); err != nil {
+			res.Err = fmt.Errorf("reconcile: bring-up data-volume resize task (%s): %w", mount, err)
+			return
+		}
+	}
+
 	// Capture the post-reset MAC for the result (fresh for provision; archived for DR keep).
 	if cfg2, err := e.api.GuestConfig(ctx, spec.VMID); err == nil {
 		res.AssignedMAC = net0MAC(cfg2)
@@ -310,7 +346,13 @@ func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]st
 		params["memory"] = strconv.Itoa(spec.MemoryMB)
 	}
 	for i, m := range spec.Mounts {
-		params[fmt.Sprintf("mp%d", i)] = fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint)
+		// backup=1 for data-bearing mounts: extra LXC mountpoints default to backup=0 = EXCLUDED
+		// from vzdump/PBS (storage-split B3), which would silently drop their DBs from the snapshot.
+		spec := fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint)
+		if m.Backup {
+			spec += ",backup=1"
+		}
+		params[fmt.Sprintf("mp%d", i)] = spec
 	}
 	return params
 }
@@ -74,6 +74,58 @@ func TestRunBringUp_ProvisionHappyPath(t *testing.T) {
 	}
 }

+// A data-bearing additive mount must carry backup=1 (so its DBs stay in PBS — storage-split B3);
+// a non-backup mount must NOT. Pure-function check on buildBringUpConfig.
+func TestBuildBringUpConfig_BackupFlagOnDataMount(t *testing.T) {
+	params := buildBringUpConfig(BringUpSpec{
+		Mode: ModeProvision,
+		Mounts: []GuestMount{
+			{Storage: "local-lvm", SizeGB: 2, MountPoint: "/mnt/data", Backup: true},
+			{Storage: "local-lvm", SizeGB: 1, MountPoint: "/mnt/scratch"}, // no backup
+		},
+	}, scratchCfg())
+	if params["mp0"] != "local-lvm:2,mp=/mnt/data,backup=1" {
+		t.Errorf("data mount must carry backup=1: mp0=%q", params["mp0"])
+	}
+	if params["mp1"] != "local-lvm:1,mp=/mnt/scratch" {
+		t.Errorf("non-backup mount must NOT carry backup=1: mp1=%q", params["mp1"])
+	}
+}
+
+// The golden-carried Docker-data volume is grown via a SEPARATE resize on its mpN slot (B4),
+// alongside (but distinct from) the rootfs grow.
+func TestRunBringUp_StorageSplit_DataVolGrow(t *testing.T) {
+	const vmid = 8050
+	api := &fakeAPI{cfg: map[int]proxmox.GuestConfig{vmid: scratchCfg()}}
+	e, _, q := newEngine(t, api, EmptyProvider{})
+	defer q.Close()
+
+	res := e.RunBringUp(context.Background(), BringUpSpec{
+		Mode: ModeProvision, Archive: "local:backup/golden.tar.zst", VMID: vmid,
+		RestoreStorage: "local-lvm", Hostname: "felhom-prov-8050",
+		RootfsGrowGB: 8, DataVolGrowGB: 240, // grows mp0 (DefaultDataVolMount)
+	})
+	if res.Err != nil || !res.Pass {
+		t.Fatalf("provision must pass, got %+v", res)
+	}
+	// TWO resizes: rootfs +8G and the Docker-data volume mp0 +240G.
+	if len(api.resizes) != 2 {
+		t.Fatalf("expected rootfs + data-volume resizes, got %+v", api.resizes)
+	}
+	var sawRootfs, sawData bool
+	for _, r := range api.resizes {
+		if r.disk == "rootfs" && r.size == "+8G" {
+			sawRootfs = true
+		}
+		if r.disk == "mp0" && r.size == "+240G" {
+			sawData = true
+		}
+	}
+	if !sawRootfs || !sawData {
+		t.Errorf("want rootfs +8G AND mp0 +240G, got %+v", api.resizes)
+	}
+}
+
 func TestRunBringUp_CompensatingRollback(t *testing.T) {
 	const vmid = 8000
 	lockBackoffFast(t)