v0.15.5: Disaster recovery — Hub-based infra backup, auto-mount, restore UI

Complete DR implementation (TASK2.md Phases 1-4): - Hub infra-backup push/pull endpoints (controller.yaml, disk layout, stacks) - Fresh-deployment detection pulls config from Hub, auto-mounts drives by UUID - Full-page restore UI with drive status, app table, sequential restore - docker-setup.sh shows DR instructions when customer_id is configured New files: disk_layout.go, restore_scan.go, restore_app_linux.go, restore_drives_linux.go, infra_backup.go, infra_pull.go, handler_restore.go, restore.html Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 13:16:46 +01:00
parent 5d993b66a2
commit 6713df2186
21 changed files with 3324 additions and 9 deletions
@@ -0,0 +1,290 @@
+//go:build linux
+
+package backup
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// MountDrivesFromLayout scans block devices for disks matching the stored
+// disk layout and mounts them using the felhom two-layer mount pattern
+// (raw mount → bind mount).
+//
+// The controller container runs privileged with:
+//   - /host-dev mounted from host /dev
+//   - /host-fstab mounted from host /etc/fstab
+//   - /mnt with rshared propagation
+//
+// Returns the list of successfully mounted final mount paths.
+func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
+	if len(layout.Mounts) == 0 {
+		return nil, nil
+	}
+
+	// Get current block devices with UUIDs
+	uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("scanning block devices: %w", err)
+	}
+
+	var mounted []string
+
+	for _, dm := range layout.Mounts {
+		if dm.UUID == "" {
+			continue
+		}
+
+		// Find matching device by UUID
+		device := uuidToDevice[dm.UUID]
+		if device == "" {
+			logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
+				dm.UUID, dm.Label)
+			continue
+		}
+
+		// Check if already mounted
+		finalMount := dm.MountPoint
+		if isMountedPath(finalMount) {
+			logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
+			mounted = append(mounted, finalMount)
+			continue
+		}
+		if dm.RawMount != "" && isMountedPath(dm.RawMount) {
+			logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
+			mounted = append(mounted, finalMount)
+			continue
+		}
+
+		uuidShort := dm.UUID
+		if len(uuidShort) > 12 {
+			uuidShort = uuidShort[:12]
+		}
+		logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
+			device, uuidShort, dm.Label, finalMount)
+
+		// Mount using the appropriate pattern
+		if dm.RawMount != "" && dm.BindSubdir != "" {
+			// Two-layer HDD mount: raw → bind
+			if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
+				logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
+				continue
+			}
+		} else {
+			// Simple direct mount (e.g., sys_drive)
+			if err := mountDirect(ctx, device, dm, logger); err != nil {
+				logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
+				continue
+			}
+		}
+
+		// Update host fstab so mount persists across reboots
+		if err := addDRFstabEntries(dm, logger); err != nil {
+			logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
+		}
+
+		mounted = append(mounted, finalMount)
+		logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
+	}
+
+	return mounted, nil
+}
+
+// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID → device path map.
+func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
+	// First try lsblk with UUID output
+	out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
+	if err != nil {
+		return nil, fmt.Errorf("lsblk failed: %w", err)
+	}
+
+	var parsed struct {
+		BlockDevices []struct {
+			Name     string  `json:"name"`
+			UUID     *string `json:"uuid"`
+			FSType   *string `json:"fstype"`
+			Mount    *string `json:"mountpoint"`
+			Children []struct {
+				Name   string  `json:"name"`
+				UUID   *string `json:"uuid"`
+				FSType *string `json:"fstype"`
+				Mount  *string `json:"mountpoint"`
+			} `json:"children"`
+		} `json:"blockdevices"`
+	}
+	if err := json.Unmarshal(out, &parsed); err != nil {
+		return nil, fmt.Errorf("lsblk parse failed: %w", err)
+	}
+
+	devices := make(map[string]string) // UUID → /dev/path
+	for _, dev := range parsed.BlockDevices {
+		if dev.UUID != nil && *dev.UUID != "" {
+			devices[*dev.UUID] = "/dev/" + dev.Name
+		}
+		for _, child := range dev.Children {
+			if child.UUID != nil && *child.UUID != "" {
+				devices[*child.UUID] = "/dev/" + child.Name
+			}
+		}
+	}
+
+	// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
+	if len(devices) == 0 {
+		// Try blkid on /host-dev devices
+		blkOut, err := exec.CommandContext(ctx, "blkid").Output()
+		if err == nil {
+			for _, line := range strings.Split(string(blkOut), "\n") {
+				line = strings.TrimSpace(line)
+				if line == "" {
+					continue
+				}
+				// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
+				colonIdx := strings.Index(line, ":")
+				if colonIdx < 0 {
+					continue
+				}
+				devPath := line[:colonIdx]
+				if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
+					rest := line[uuidIdx+6:]
+					if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
+						uuid := rest[:endIdx]
+						devices[uuid] = devPath
+					}
+				}
+			}
+		}
+	}
+
+	return devices, nil
+}
+
+// mountDirect creates a simple direct mount.
+func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
+	if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
+		return fmt.Errorf("creating mount point: %w", err)
+	}
+
+	// Use host device path if available
+	devPath := hostDevPath(device)
+	cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
+	}
+	return nil
+}
+
+// mountRawAndBind implements the two-layer felhom mount pattern.
+func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
+	// Layer 1: raw mount
+	if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
+		return fmt.Errorf("creating raw mount point: %w", err)
+	}
+
+	devPath := hostDevPath(device)
+	cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("raw mount %s → %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
+	}
+
+	// Layer 2: bind mount (subdir → final mount point)
+	bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
+	if err := os.MkdirAll(bindSrc, 0755); err != nil {
+		return fmt.Errorf("creating bind source dir: %w", err)
+	}
+	if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
+		return fmt.Errorf("creating final mount point: %w", err)
+	}
+
+	cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("bind mount %s → %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
+	}
+
+	return nil
+}
+
+// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
+func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
+	const fstabPath = "/host-fstab"
+
+	data, err := os.ReadFile(fstabPath)
+	if err != nil {
+		return fmt.Errorf("reading fstab: %w", err)
+	}
+
+	content := string(data)
+
+	// Skip if UUID already in fstab (idempotent)
+	if strings.Contains(content, dm.UUID) {
+		return nil
+	}
+
+	var additions strings.Builder
+	additions.WriteString("\n# Restored by felhom-controller DR\n")
+
+	if dm.RawMount != "" {
+		// Raw mount entry
+		additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
+			dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
+	}
+
+	if dm.BindSubdir != "" && dm.RawMount != "" {
+		// Bind mount entry
+		additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
+			dm.RawMount, dm.BindSubdir, dm.MountPoint))
+	} else if dm.RawMount == "" {
+		// Direct mount entry (no bind)
+		additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
+			dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
+	}
+
+	newContent := content + additions.String()
+
+	// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
+	tmpPath := fstabPath + ".tmp"
+	if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
+		return fmt.Errorf("writing fstab tmp: %w", err)
+	}
+	if err := os.Rename(tmpPath, fstabPath); err != nil {
+		os.Remove(tmpPath)
+		// Fallback: direct write (bind-mounted files can't be renamed)
+		if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
+			return fmt.Errorf("writing fstab: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// isMountedPath checks if a path is currently a mount point via /proc/mounts.
+func isMountedPath(path string) bool {
+	if path == "" {
+		return false
+	}
+	data, err := os.ReadFile("/proc/mounts")
+	if err != nil {
+		return false
+	}
+	cleanPath := filepath.Clean(path)
+	for _, line := range strings.Split(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
+			return true
+		}
+	}
+	return false
+}
+
+// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
+func hostDevPath(devPath string) string {
+	if strings.HasPrefix(devPath, "/dev/") {
+		return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
+	}
+	return devPath
+}