Files
deploy-felhom-compose/controller/internal/backup/restore_drives_linux.go
T
admin 6713df2186 v0.15.5: Disaster recovery — Hub-based infra backup, auto-mount, restore UI
Complete DR implementation (TASK2.md Phases 1-4):
- Hub infra-backup push/pull endpoints (controller.yaml, disk layout, stacks)
- Fresh-deployment detection pulls config from Hub, auto-mounts drives by UUID
- Full-page restore UI with drive status, app table, sequential restore
- docker-setup.sh shows DR instructions when customer_id is configured

New files: disk_layout.go, restore_scan.go, restore_app_linux.go,
restore_drives_linux.go, infra_backup.go, infra_pull.go,
handler_restore.go, restore.html

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 13:16:46 +01:00

291 lines
8.6 KiB
Go

//go:build linux
package backup
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
// MountDrivesFromLayout scans block devices for disks matching the stored
// disk layout and mounts them using the felhom two-layer mount pattern
// (raw mount → bind mount).
//
// The controller container runs privileged with:
// - /host-dev mounted from host /dev
// - /host-fstab mounted from host /etc/fstab
// - /mnt with rshared propagation
//
// Returns the list of successfully mounted final mount paths.
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
if len(layout.Mounts) == 0 {
return nil, nil
}
// Get current block devices with UUIDs
uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
if err != nil {
return nil, fmt.Errorf("scanning block devices: %w", err)
}
var mounted []string
for _, dm := range layout.Mounts {
if dm.UUID == "" {
continue
}
// Find matching device by UUID
device := uuidToDevice[dm.UUID]
if device == "" {
logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
dm.UUID, dm.Label)
continue
}
// Check if already mounted
finalMount := dm.MountPoint
if isMountedPath(finalMount) {
logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
mounted = append(mounted, finalMount)
continue
}
if dm.RawMount != "" && isMountedPath(dm.RawMount) {
logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
mounted = append(mounted, finalMount)
continue
}
uuidShort := dm.UUID
if len(uuidShort) > 12 {
uuidShort = uuidShort[:12]
}
logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
device, uuidShort, dm.Label, finalMount)
// Mount using the appropriate pattern
if dm.RawMount != "" && dm.BindSubdir != "" {
// Two-layer HDD mount: raw → bind
if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
} else {
// Simple direct mount (e.g., sys_drive)
if err := mountDirect(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
}
// Update host fstab so mount persists across reboots
if err := addDRFstabEntries(dm, logger); err != nil {
logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
}
mounted = append(mounted, finalMount)
logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
}
return mounted, nil
}
// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID → device path map.
func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
// First try lsblk with UUID output
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
if err != nil {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
var parsed struct {
BlockDevices []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
Children []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
} `json:"children"`
} `json:"blockdevices"`
}
if err := json.Unmarshal(out, &parsed); err != nil {
return nil, fmt.Errorf("lsblk parse failed: %w", err)
}
devices := make(map[string]string) // UUID → /dev/path
for _, dev := range parsed.BlockDevices {
if dev.UUID != nil && *dev.UUID != "" {
devices[*dev.UUID] = "/dev/" + dev.Name
}
for _, child := range dev.Children {
if child.UUID != nil && *child.UUID != "" {
devices[*child.UUID] = "/dev/" + child.Name
}
}
}
// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
if len(devices) == 0 {
// Try blkid on /host-dev devices
blkOut, err := exec.CommandContext(ctx, "blkid").Output()
if err == nil {
for _, line := range strings.Split(string(blkOut), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
colonIdx := strings.Index(line, ":")
if colonIdx < 0 {
continue
}
devPath := line[:colonIdx]
if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
rest := line[uuidIdx+6:]
if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
uuid := rest[:endIdx]
devices[uuid] = devPath
}
}
}
}
}
return devices, nil
}
// mountDirect creates a simple direct mount.
func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating mount point: %w", err)
}
// Use host device path if available
devPath := hostDevPath(device)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
}
return nil
}
// mountRawAndBind implements the two-layer felhom mount pattern.
func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
// Layer 1: raw mount
if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
return fmt.Errorf("creating raw mount point: %w", err)
}
devPath := hostDevPath(device)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("raw mount %s → %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
}
// Layer 2: bind mount (subdir → final mount point)
bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
if err := os.MkdirAll(bindSrc, 0755); err != nil {
return fmt.Errorf("creating bind source dir: %w", err)
}
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating final mount point: %w", err)
}
cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("bind mount %s → %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
}
return nil
}
// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
const fstabPath = "/host-fstab"
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("reading fstab: %w", err)
}
content := string(data)
// Skip if UUID already in fstab (idempotent)
if strings.Contains(content, dm.UUID) {
return nil
}
var additions strings.Builder
additions.WriteString("\n# Restored by felhom-controller DR\n")
if dm.RawMount != "" {
// Raw mount entry
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
}
if dm.BindSubdir != "" && dm.RawMount != "" {
// Bind mount entry
additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
dm.RawMount, dm.BindSubdir, dm.MountPoint))
} else if dm.RawMount == "" {
// Direct mount entry (no bind)
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
}
newContent := content + additions.String()
// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
tmpPath := fstabPath + ".tmp"
if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab tmp: %w", err)
}
if err := os.Rename(tmpPath, fstabPath); err != nil {
os.Remove(tmpPath)
// Fallback: direct write (bind-mounted files can't be renamed)
if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab: %w", err)
}
}
return nil
}
// isMountedPath checks if a path is currently a mount point via /proc/mounts.
func isMountedPath(path string) bool {
if path == "" {
return false
}
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false
}
cleanPath := filepath.Clean(path)
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
return true
}
}
return false
}
// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
func hostDevPath(devPath string) string {
if strings.HasPrefix(devPath, "/dev/") {
return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
}
return devPath
}