8e61cd7ec4
Add structured operational logging at INFO, WARN, and ERROR levels to every controller module. Standardize custom prefixes ([GEO], [SCHED], [SYNC]) to use [INFO/WARN/ERROR] [module] format. Fix misleveled logs (WARN->ERROR for data loss scenarios, WARN->INFO for routine operations). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
326 lines
11 KiB
Go
326 lines
11 KiB
Go
//go:build linux
|
|
|
|
package backup
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
// MountDrivesFromLayout scans block devices for disks matching the stored
|
|
// disk layout and mounts them using the felhom two-layer mount pattern
|
|
// (raw mount → bind mount).
|
|
//
|
|
// The controller container runs privileged with:
|
|
// - /host-dev mounted from host /dev
|
|
// - /host-fstab mounted from host /etc/fstab
|
|
// - /mnt with rshared propagation
|
|
//
|
|
// Returns the list of successfully mounted final mount paths.
|
|
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
|
|
if len(layout.Mounts) == 0 {
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: no mounts in layout, nothing to do")
|
|
return nil, nil
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %d mount entries from disk layout", len(layout.Mounts))
|
|
|
|
// Get current block devices with UUIDs
|
|
uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scanning block devices: %w", err)
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: discovered %d block devices with UUIDs", len(uuidToDevice))
|
|
for uuid, dev := range uuidToDevice {
|
|
uuidShort := uuid
|
|
if len(uuidShort) > 12 {
|
|
uuidShort = uuidShort[:12]
|
|
}
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: device %s → UUID=%s...", dev, uuidShort)
|
|
}
|
|
|
|
var mounted []string
|
|
|
|
for _, dm := range layout.Mounts {
|
|
if dm.UUID == "" {
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: skipping mount entry with empty UUID (label=%s)", dm.Label)
|
|
continue
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %s (UUID=%s, mountPoint=%s, rawMount=%s, fsType=%s)",
|
|
dm.Label, dm.UUID, dm.MountPoint, dm.RawMount, dm.FSType)
|
|
|
|
// Find matching device by UUID
|
|
device := uuidToDevice[dm.UUID]
|
|
if device == "" {
|
|
logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
|
|
dm.UUID, dm.Label)
|
|
continue
|
|
}
|
|
|
|
// Check if already mounted
|
|
finalMount := dm.MountPoint
|
|
if isMountedPath(finalMount) {
|
|
logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
|
|
mounted = append(mounted, finalMount)
|
|
continue
|
|
}
|
|
if dm.RawMount != "" && isMountedPath(dm.RawMount) {
|
|
logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
|
|
mounted = append(mounted, finalMount)
|
|
continue
|
|
}
|
|
|
|
uuidShort := dm.UUID
|
|
if len(uuidShort) > 12 {
|
|
uuidShort = uuidShort[:12]
|
|
}
|
|
logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
|
|
device, uuidShort, dm.Label, finalMount)
|
|
|
|
// Mount using the appropriate pattern
|
|
if dm.RawMount != "" && dm.BindSubdir != "" {
|
|
// Two-layer HDD mount: raw → bind
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — two-layer mount (raw=%s, bindSubdir=%s)",
|
|
dm.Label, dm.RawMount, dm.BindSubdir)
|
|
if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
|
|
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
|
|
continue
|
|
}
|
|
} else {
|
|
// Simple direct mount (e.g., sys_drive)
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — direct mount to %s", dm.Label, dm.MountPoint)
|
|
if err := mountDirect(ctx, device, dm, logger); err != nil {
|
|
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Update host fstab so mount persists across reboots
|
|
if err := addDRFstabEntries(dm, logger); err != nil {
|
|
logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
|
|
}
|
|
|
|
mounted = append(mounted, finalMount)
|
|
logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: done — %d/%d drives mounted", len(mounted), len(layout.Mounts))
|
|
return mounted, nil
|
|
}
|
|
|
|
// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID -> device path map.
|
|
func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
|
|
// First try lsblk with UUID output
|
|
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("lsblk failed: %w", err)
|
|
}
|
|
|
|
var parsed struct {
|
|
BlockDevices []struct {
|
|
Name string `json:"name"`
|
|
UUID *string `json:"uuid"`
|
|
FSType *string `json:"fstype"`
|
|
Mount *string `json:"mountpoint"`
|
|
Children []struct {
|
|
Name string `json:"name"`
|
|
UUID *string `json:"uuid"`
|
|
FSType *string `json:"fstype"`
|
|
Mount *string `json:"mountpoint"`
|
|
} `json:"children"`
|
|
} `json:"blockdevices"`
|
|
}
|
|
if err := json.Unmarshal(out, &parsed); err != nil {
|
|
return nil, fmt.Errorf("lsblk parse failed: %w", err)
|
|
}
|
|
|
|
devices := make(map[string]string) // UUID → /dev/path
|
|
for _, dev := range parsed.BlockDevices {
|
|
if dev.UUID != nil && *dev.UUID != "" {
|
|
devices[*dev.UUID] = "/dev/" + dev.Name
|
|
}
|
|
for _, child := range dev.Children {
|
|
if child.UUID != nil && *child.UUID != "" {
|
|
devices[*child.UUID] = "/dev/" + child.Name
|
|
}
|
|
}
|
|
}
|
|
|
|
// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
|
|
if len(devices) == 0 {
|
|
// Try blkid on /host-dev devices
|
|
blkOut, err := exec.CommandContext(ctx, "blkid").Output()
|
|
if err == nil {
|
|
for _, line := range strings.Split(string(blkOut), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
|
|
colonIdx := strings.Index(line, ":")
|
|
if colonIdx < 0 {
|
|
continue
|
|
}
|
|
devPath := line[:colonIdx]
|
|
if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
|
|
rest := line[uuidIdx+6:]
|
|
if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
|
|
uuid := rest[:endIdx]
|
|
devices[uuid] = devPath
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return devices, nil
|
|
}
|
|
|
|
// mountDirect creates a simple direct mount.
|
|
func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
|
|
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
|
|
return fmt.Errorf("creating mount point: %w", err)
|
|
}
|
|
|
|
// Use host device path if available
|
|
devPath := hostDevPath(device)
|
|
logger.Printf("[DEBUG] [backup] mountDirect: mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.MountPoint)
|
|
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
|
|
}
|
|
logger.Printf("[DEBUG] [backup] mountDirect: %s mounted successfully at %s", devPath, dm.MountPoint)
|
|
return nil
|
|
}
|
|
|
|
// mountRawAndBind implements the two-layer felhom mount pattern.
|
|
func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
|
|
// Layer 1: raw mount
|
|
if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
|
|
return fmt.Errorf("creating raw mount point: %w", err)
|
|
}
|
|
|
|
devPath := hostDevPath(device)
|
|
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 — mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.RawMount)
|
|
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("raw mount %s -> %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
|
|
}
|
|
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 OK — %s mounted at %s", devPath, dm.RawMount)
|
|
|
|
// Layer 2: bind mount (subdir -> final mount point)
|
|
bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
|
|
if err := os.MkdirAll(bindSrc, 0755); err != nil {
|
|
return fmt.Errorf("creating bind source dir: %w", err)
|
|
}
|
|
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
|
|
return fmt.Errorf("creating final mount point: %w", err)
|
|
}
|
|
|
|
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 — mount --bind %s %s", bindSrc, dm.MountPoint)
|
|
cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("bind mount %s -> %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
|
|
}
|
|
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 OK — %s bound to %s", bindSrc, dm.MountPoint)
|
|
|
|
return nil
|
|
}
|
|
|
|
// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
|
|
func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
|
|
const fstabPath = "/host-fstab"
|
|
|
|
logger.Printf("[INFO] [backup] Adding fstab entries for disaster recovery (%s, UUID=%s)", dm.Label, dm.UUID)
|
|
logger.Printf("[DEBUG] [backup] addDRFstabEntries: checking fstab for %s (UUID=%s)", dm.Label, dm.UUID)
|
|
|
|
data, err := os.ReadFile(fstabPath)
|
|
if err != nil {
|
|
return fmt.Errorf("reading fstab: %w", err)
|
|
}
|
|
|
|
content := string(data)
|
|
|
|
// Skip if UUID already in fstab (idempotent)
|
|
if strings.Contains(content, dm.UUID) {
|
|
logger.Printf("[DEBUG] [backup] addDRFstabEntries: UUID %s already in fstab — skipping", dm.UUID)
|
|
return nil
|
|
}
|
|
|
|
var additions strings.Builder
|
|
additions.WriteString("\n# Restored by felhom-controller DR\n")
|
|
|
|
entryCount := 0
|
|
if dm.RawMount != "" {
|
|
// Raw mount entry
|
|
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
|
|
dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
|
|
entryCount++
|
|
}
|
|
|
|
if dm.BindSubdir != "" && dm.RawMount != "" {
|
|
// Bind mount entry
|
|
additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
|
|
dm.RawMount, dm.BindSubdir, dm.MountPoint))
|
|
entryCount++
|
|
} else if dm.RawMount == "" {
|
|
// Direct mount entry (no bind)
|
|
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
|
|
dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
|
|
entryCount++
|
|
}
|
|
|
|
newContent := content + additions.String()
|
|
|
|
// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
|
|
tmpPath := fstabPath + ".tmp"
|
|
if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
|
|
return fmt.Errorf("writing fstab tmp: %w", err)
|
|
}
|
|
if err := os.Rename(tmpPath, fstabPath); err != nil {
|
|
os.Remove(tmpPath)
|
|
// Fallback: direct write (bind-mounted files can't be renamed)
|
|
if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
|
|
return fmt.Errorf("writing fstab: %w", err)
|
|
}
|
|
}
|
|
|
|
logger.Printf("[INFO] [backup] Added %d fstab entries for %s", entryCount, dm.Label)
|
|
return nil
|
|
}
|
|
|
|
// isMountedPath checks if a path is currently a mount point via /proc/mounts.
|
|
func isMountedPath(path string) bool {
|
|
if path == "" {
|
|
return false
|
|
}
|
|
data, err := os.ReadFile("/proc/mounts")
|
|
if err != nil {
|
|
return false
|
|
}
|
|
cleanPath := filepath.Clean(path)
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
|
|
func hostDevPath(devPath string) string {
|
|
if strings.HasPrefix(devPath, "/dev/") {
|
|
return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
|
|
}
|
|
return devPath
|
|
}
|