Files
admin 8e61cd7ec4 feat: comprehensive INFO/WARN/ERROR logging across all controller modules
Add structured operational logging at INFO, WARN, and ERROR levels to
every controller module. Standardize custom prefixes ([GEO], [SCHED],
[SYNC]) to use [INFO/WARN/ERROR] [module] format. Fix misleveled logs
(WARN->ERROR for data loss scenarios, WARN->INFO for routine operations).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 19:58:27 +01:00

326 lines
11 KiB
Go

//go:build linux
package backup
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
// MountDrivesFromLayout scans block devices for disks matching the stored
// disk layout and mounts them using the felhom two-layer mount pattern
// (raw mount → bind mount).
//
// The controller container runs privileged with:
// - /host-dev mounted from host /dev
// - /host-fstab mounted from host /etc/fstab
// - /mnt with rshared propagation
//
// Returns the list of successfully mounted final mount paths.
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
if len(layout.Mounts) == 0 {
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: no mounts in layout, nothing to do")
return nil, nil
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %d mount entries from disk layout", len(layout.Mounts))
// Get current block devices with UUIDs
uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
if err != nil {
return nil, fmt.Errorf("scanning block devices: %w", err)
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: discovered %d block devices with UUIDs", len(uuidToDevice))
for uuid, dev := range uuidToDevice {
uuidShort := uuid
if len(uuidShort) > 12 {
uuidShort = uuidShort[:12]
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: device %s → UUID=%s...", dev, uuidShort)
}
var mounted []string
for _, dm := range layout.Mounts {
if dm.UUID == "" {
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: skipping mount entry with empty UUID (label=%s)", dm.Label)
continue
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %s (UUID=%s, mountPoint=%s, rawMount=%s, fsType=%s)",
dm.Label, dm.UUID, dm.MountPoint, dm.RawMount, dm.FSType)
// Find matching device by UUID
device := uuidToDevice[dm.UUID]
if device == "" {
logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
dm.UUID, dm.Label)
continue
}
// Check if already mounted
finalMount := dm.MountPoint
if isMountedPath(finalMount) {
logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
mounted = append(mounted, finalMount)
continue
}
if dm.RawMount != "" && isMountedPath(dm.RawMount) {
logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
mounted = append(mounted, finalMount)
continue
}
uuidShort := dm.UUID
if len(uuidShort) > 12 {
uuidShort = uuidShort[:12]
}
logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
device, uuidShort, dm.Label, finalMount)
// Mount using the appropriate pattern
if dm.RawMount != "" && dm.BindSubdir != "" {
// Two-layer HDD mount: raw → bind
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — two-layer mount (raw=%s, bindSubdir=%s)",
dm.Label, dm.RawMount, dm.BindSubdir)
if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
} else {
// Simple direct mount (e.g., sys_drive)
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — direct mount to %s", dm.Label, dm.MountPoint)
if err := mountDirect(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
}
// Update host fstab so mount persists across reboots
if err := addDRFstabEntries(dm, logger); err != nil {
logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
}
mounted = append(mounted, finalMount)
logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: done — %d/%d drives mounted", len(mounted), len(layout.Mounts))
return mounted, nil
}
// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID -> device path map.
func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
// First try lsblk with UUID output
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
if err != nil {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
var parsed struct {
BlockDevices []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
Children []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
} `json:"children"`
} `json:"blockdevices"`
}
if err := json.Unmarshal(out, &parsed); err != nil {
return nil, fmt.Errorf("lsblk parse failed: %w", err)
}
devices := make(map[string]string) // UUID → /dev/path
for _, dev := range parsed.BlockDevices {
if dev.UUID != nil && *dev.UUID != "" {
devices[*dev.UUID] = "/dev/" + dev.Name
}
for _, child := range dev.Children {
if child.UUID != nil && *child.UUID != "" {
devices[*child.UUID] = "/dev/" + child.Name
}
}
}
// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
if len(devices) == 0 {
// Try blkid on /host-dev devices
blkOut, err := exec.CommandContext(ctx, "blkid").Output()
if err == nil {
for _, line := range strings.Split(string(blkOut), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
colonIdx := strings.Index(line, ":")
if colonIdx < 0 {
continue
}
devPath := line[:colonIdx]
if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
rest := line[uuidIdx+6:]
if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
uuid := rest[:endIdx]
devices[uuid] = devPath
}
}
}
}
}
return devices, nil
}
// mountDirect creates a simple direct mount.
func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating mount point: %w", err)
}
// Use host device path if available
devPath := hostDevPath(device)
logger.Printf("[DEBUG] [backup] mountDirect: mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.MountPoint)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountDirect: %s mounted successfully at %s", devPath, dm.MountPoint)
return nil
}
// mountRawAndBind implements the two-layer felhom mount pattern.
func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
// Layer 1: raw mount
if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
return fmt.Errorf("creating raw mount point: %w", err)
}
devPath := hostDevPath(device)
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 — mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.RawMount)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("raw mount %s -> %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 OK — %s mounted at %s", devPath, dm.RawMount)
// Layer 2: bind mount (subdir -> final mount point)
bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
if err := os.MkdirAll(bindSrc, 0755); err != nil {
return fmt.Errorf("creating bind source dir: %w", err)
}
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating final mount point: %w", err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 — mount --bind %s %s", bindSrc, dm.MountPoint)
cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("bind mount %s -> %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 OK — %s bound to %s", bindSrc, dm.MountPoint)
return nil
}
// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
const fstabPath = "/host-fstab"
logger.Printf("[INFO] [backup] Adding fstab entries for disaster recovery (%s, UUID=%s)", dm.Label, dm.UUID)
logger.Printf("[DEBUG] [backup] addDRFstabEntries: checking fstab for %s (UUID=%s)", dm.Label, dm.UUID)
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("reading fstab: %w", err)
}
content := string(data)
// Skip if UUID already in fstab (idempotent)
if strings.Contains(content, dm.UUID) {
logger.Printf("[DEBUG] [backup] addDRFstabEntries: UUID %s already in fstab — skipping", dm.UUID)
return nil
}
var additions strings.Builder
additions.WriteString("\n# Restored by felhom-controller DR\n")
entryCount := 0
if dm.RawMount != "" {
// Raw mount entry
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
entryCount++
}
if dm.BindSubdir != "" && dm.RawMount != "" {
// Bind mount entry
additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
dm.RawMount, dm.BindSubdir, dm.MountPoint))
entryCount++
} else if dm.RawMount == "" {
// Direct mount entry (no bind)
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
entryCount++
}
newContent := content + additions.String()
// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
tmpPath := fstabPath + ".tmp"
if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab tmp: %w", err)
}
if err := os.Rename(tmpPath, fstabPath); err != nil {
os.Remove(tmpPath)
// Fallback: direct write (bind-mounted files can't be renamed)
if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab: %w", err)
}
}
logger.Printf("[INFO] [backup] Added %d fstab entries for %s", entryCount, dm.Label)
return nil
}
// isMountedPath checks if a path is currently a mount point via /proc/mounts.
func isMountedPath(path string) bool {
if path == "" {
return false
}
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false
}
cleanPath := filepath.Clean(path)
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
return true
}
}
return false
}
// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
func hostDevPath(devPath string) string {
if strings.HasPrefix(devPath, "/dev/") {
return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
}
return devPath
}