v0.24.0 — Pre-testing observability: debug logging, diagnostic dump, startup self-test

- Add [DEBUG] logging across all modules (backup, storage, sync, selfupdate,
  monitor, notify, report, assets, setup) gated behind logging.level: "debug"
- Add /api/debug/dump endpoint returning full controller state JSON (debug only)
- Add startup self-test validating 9 subsystems (Docker, dirs, storage, hub,
  restic repos, metrics DB) with pass/warn/fail summary
- New packages: internal/selftest, internal/util
- Constructor/signature changes: debug bool params, logger params on
  RunHealthCheck and BuildReport, smart watchdog probe logging

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-21 18:32:26 +01:00
parent 6f02536243
commit be7803c0ac
30 changed files with 1281 additions and 67 deletions
+9 -5
View File
@@ -1,13 +1,17 @@
package storage
import "log"
// AttachRequest holds parameters for attaching an existing partition
// without formatting — using a bind mount from a subfolder.
type AttachRequest struct {
DevicePath string // "/dev/sdb1" — must have an existing filesystem
MountName string // "hdd_1" → bind-mounts at /mnt/hdd_1
SubPath string // full path on raw mount to bind-mount (e.g., "/mnt/.felhom-raw/hdd_1/felhom_data")
Label string // Display label for the UI
SetDefault bool // Register as default storage path
DevicePath string // "/dev/sdb1" — must have an existing filesystem
MountName string // "hdd_1" → bind-mounts at /mnt/hdd_1
SubPath string // full path on raw mount to bind-mount (e.g., "/mnt/.felhom-raw/hdd_1/felhom_data")
Label string // Display label for the UI
SetDefault bool // Register as default storage path
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// DirEntry represents a directory entry returned by ListDirectories.
@@ -182,8 +182,14 @@ func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string,
progress <- FormatProgress{Step: "error", Message: msg, Error: errStr, Percent: 0}
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] FinalizeAttach: "+format, args...)
}
}
mountPath := "/mnt/" + req.MountName
dbg("starting: device=%s mountName=%s subPath=%s", req.DevicePath, req.MountName, req.SubPath)
// --- Step 1: Validate ---
send("validating", "Paraméterek ellenőrzése...", 5)
@@ -244,6 +250,7 @@ func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string,
subRel := strings.TrimPrefix(cleanSub, rawMountPath)
subRel = strings.TrimPrefix(subRel, "/")
dbg("raw mount path: %s, sub relative: %q", rawMountPath, subRel)
send("mounting", "fstab bejegyzések hozzáadása...", 35)
// Backup fstab (non-fatal)
@@ -251,13 +258,17 @@ func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string,
// Fstab entry 1: raw partition mount
// Use nofail so a missing disk doesn't block boot
dbg("fstab entry 1: UUID=%s → %s (fstype=%s)", uuid, rawMountPath, fsType)
if err := AppendFstabEntry(FstabPath, uuid, rawMountPath, fsType, "defaults,nofail,noatime"); err != nil {
dbg("fstab raw mount entry failed: %v", err)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen (raw mount)", err)
}
// Fstab entry 2: bind mount from subfolder to final path
bindSource := cleanSub
dbg("fstab entry 2: bind %s → %s", bindSource, mountPath)
if err := appendBindFstabEntry(FstabPath, bindSource, mountPath); err != nil {
dbg("fstab bind entry failed: %v", err)
// Roll back the raw mount fstab entry
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen (bind mount)", err)
@@ -272,20 +283,25 @@ func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string,
return "", fail("mounting", "Csatlakoztatási mappa nem hozható létre: "+mountPath, err)
}
dbg("bind mount: mount --bind %s %s", cleanSub, mountPath)
if out, err := exec.Command("mount", "--bind", cleanSub, mountPath).CombinedOutput(); err != nil {
dbg("bind mount failed: %s", string(out))
_ = RemoveFstabEntry(FstabPath, uuid)
_ = removeBindFstabEntry(FstabPath, mountPath)
return "", fail("mounting", "Bind mount sikertelen: "+string(out), err)
}
// Verify bind mount
dbg("verifying bind mount with findmnt")
verifyOut, verifyErr := exec.Command("findmnt", "-n", "-o", "SOURCE", "--target", mountPath).Output()
if verifyErr != nil || strings.TrimSpace(string(verifyOut)) == "" {
dbg("bind mount verification failed: findmnt returned %q err=%v", string(verifyOut), verifyErr)
_ = exec.Command("umount", mountPath).Run()
_ = RemoveFstabEntry(FstabPath, uuid)
_ = removeBindFstabEntry(FstabPath, mountPath)
return "", fail("mounting", "A bind mount nem ellenőrizhető", fmt.Errorf("mount point %s not found after bind mount", mountPath))
}
dbg("bind mount verified: source=%q", strings.TrimSpace(string(verifyOut)))
send("mounting", "Csatlakoztatva: "+mountPath, 70)
@@ -301,6 +317,7 @@ func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string,
}
}
dbg("attach completed successfully: %s", mountPath)
send("done", "Meghajtó sikeresen csatolva: "+mountPath, 100)
return mountPath, nil
+8 -5
View File
@@ -3,16 +3,19 @@ package storage
import (
"bufio"
"fmt"
"log"
"strings"
)
// FormatRequest holds parameters for formatting and mounting a disk.
type FormatRequest struct {
DevicePath string // "/dev/sdb" or "/dev/sdb1"
MountName string // "hdd_1" → mounts at /mnt/hdd_1
Label string // Display label for the UI
CreatePartition bool // If true, create a single partition first (wipes disk)
SetDefault bool // Register as default storage path
DevicePath string // "/dev/sdb" or "/dev/sdb1"
MountName string // "hdd_1" → mounts at /mnt/hdd_1
Label string // Display label for the UI
CreatePartition bool // If true, create a single partition first (wipes disk)
SetDefault bool // Register as default storage path
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// FormatProgress tracks the formatting/mounting progress.
@@ -10,6 +10,8 @@ import (
"path/filepath"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// FormatAndMount formats a disk/partition and mounts it.
@@ -27,8 +29,14 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
progress <- FormatProgress{Step: "error", Message: msg, Error: errStr, Percent: 0}
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] FormatAndMount: "+format, args...)
}
}
mountPath := "/mnt/" + req.MountName
dbg("starting: device=%s mountName=%s createPartition=%v", req.DevicePath, req.MountName, req.CreatePartition)
// --- Step 1: Validate ---
send("validating", "Eszköz ellenőrzése...", 5)
@@ -78,10 +86,14 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
if req.CreatePartition {
// Wipe existing partition table and filesystem signatures first
// H18: Log wipefs errors instead of silently discarding them.
dbg("step wipefs: wipefs -a %s", HostDevicePath(req.DevicePath))
send("partitioning", fmt.Sprintf("wipefs -a %s ...", HostDevicePath(req.DevicePath)), 12)
if err := exec.Command("wipefs", "-a", HostDevicePath(req.DevicePath)).Run(); err != nil {
// Non-fatal: some systems don't have wipefs; continue anyway
dbg("wipefs failed (non-fatal): %v", err)
send("partitioning", fmt.Sprintf("[WARN] wipefs sikertelen %s: %v (folytatás)", req.DevicePath, err), 13)
} else {
dbg("wipefs completed successfully")
}
time.Sleep(500 * time.Millisecond)
@@ -89,12 +101,16 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
// ",," = start=default, size=default(fill disk), type=default(Linux filesystem GUID).
// --force: overwrite even if device appears busy.
// --wipe always: wipe filesystem signatures from newly created partitions.
dbg("step sfdisk: sfdisk --force --wipe always %s", HostDevicePath(req.DevicePath))
send("partitioning", fmt.Sprintf("sfdisk --force --wipe always %s ...", HostDevicePath(req.DevicePath)), 15)
sfdiskInput := "label: gpt\n,,\n"
cmd := exec.Command("sfdisk", "--force", "--wipe", "always", HostDevicePath(req.DevicePath))
cmd.Stdin = strings.NewReader(sfdiskInput)
if out, err := cmd.CombinedOutput(); err != nil {
dbg("sfdisk failed: %s", util.TruncateStr(string(out), 500))
return "", fail("partitioning", "Partícionálás sikertelen: "+string(out), err)
} else {
dbg("sfdisk output: %s", util.TruncateStr(string(out), 500))
}
_ = exec.Command("partprobe", HostDevicePath(req.DevicePath)).Run()
@@ -119,14 +135,17 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
fsLabel = fsLabel[:16]
}
dbg("step mkfs.ext4: mkfs.ext4 -L %s -F %s", fsLabel, HostDevicePath(partDev))
send("formatting", fmt.Sprintf("mkfs.ext4 -L %s -F %s ...", fsLabel, HostDevicePath(partDev)), 30)
mkfsCmd := exec.Command("mkfs.ext4", "-L", fsLabel, "-F", HostDevicePath(partDev))
var mkfsOut bytes.Buffer
mkfsCmd.Stdout = &mkfsOut
mkfsCmd.Stderr = &mkfsOut
if err := mkfsCmd.Run(); err != nil {
dbg("mkfs.ext4 failed: %s", util.TruncateStr(mkfsOut.String(), 500))
return "", fail("formatting", "Formázás sikertelen: "+mkfsOut.String(), err)
}
dbg("mkfs.ext4 output: %s", util.TruncateStr(mkfsOut.String(), 500))
send("formatting", "Formázás kész", 60)
@@ -135,12 +154,15 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
return "", fail("mounting", "Csatlakoztatási mappa nem hozható létre: "+mountPath, err)
}
dbg("step blkid: blkid -s UUID -o value %s", HostDevicePath(partDev))
send("mounting", fmt.Sprintf("UUID lekérése: blkid %s ...", HostDevicePath(partDev)), 65)
uuidOut, err := exec.Command("blkid", "-s", "UUID", "-o", "value", HostDevicePath(partDev)).Output()
if err != nil {
dbg("blkid UUID failed: %v", err)
return "", fail("mounting", "UUID lekérése sikertelen", err)
}
uuid := strings.TrimSpace(string(uuidOut))
dbg("blkid returned UUID=%q", uuid)
if uuid == "" {
return "", fail("mounting", "UUID üres a formázás után", fmt.Errorf("empty UUID"))
}
@@ -148,28 +170,36 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
// Backup fstab (non-fatal)
_ = BackupFstab(FstabPath)
dbg("step fstab: appending UUID=%s mountPath=%s fstype=ext4", uuid, mountPath)
if err := AppendFstabEntry(FstabPath, uuid, mountPath, "ext4", "defaults,nofail,noatime"); err != nil {
dbg("fstab append failed: %v", err)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen", err)
}
dbg("fstab entry added successfully")
// Mount by device path explicitly — container's /etc/fstab != host fstab,
// so "mount /mnt/hdd_1" (fstab lookup) won't work.
dbg("step mount: mount -t ext4 -o defaults,noatime %s %s", HostDevicePath(partDev), mountPath)
send("mounting", fmt.Sprintf("mount -t ext4 %s %s ...", HostDevicePath(partDev), mountPath), 70)
if out, err := exec.Command("mount", "-t", "ext4", "-o", "defaults,noatime",
HostDevicePath(partDev), mountPath).CombinedOutput(); err != nil {
dbg("mount failed: %s", util.TruncateStr(string(out), 500))
// H19: Roll back fstab entry to prevent orphaned entry that hangs system on reboot.
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "Csatlakoztatás sikertelen: "+string(out), err)
}
// Verify mount actually worked (don't just trust exit code)
dbg("step verify: findmnt -n -o SOURCE --target %s", mountPath)
verifyOut, verifyErr := exec.Command("findmnt", "-n", "-o", "SOURCE", "--target", mountPath).Output()
if verifyErr != nil || strings.TrimSpace(string(verifyOut)) == "" {
dbg("mount verification failed: findmnt returned %q err=%v", string(verifyOut), verifyErr)
// H19: Also roll back fstab if mount verify fails.
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "A csatlakoztatás nem ellenőrizhető: mount sikerült, de a meghajtó nem látható",
fmt.Errorf("mount point %s not found after mount", mountPath))
}
dbg("mount verified: findmnt source=%q", strings.TrimSpace(string(verifyOut)))
send("mounting", "Csatlakoztatva: "+mountPath, 80)
@@ -185,6 +215,7 @@ func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string,
}
}
dbg("format and mount completed successfully: %s", mountPath)
send("done", "Meghajtó sikeresen inicializálva: "+mountPath, 100)
return mountPath, nil
+23 -5
View File
@@ -18,11 +18,13 @@ import (
// MigrateRequest holds parameters for migrating app data.
type MigrateRequest struct {
StackName string // e.g., "immich"
DisplayName string // e.g., "Immich"
CurrentHDDPath string // e.g., "/mnt/hdd_placeholder"
TargetPath string // e.g., "/mnt/hdd_1"
HDDMounts []string // host-side paths to rsync (e.g., ["/mnt/hdd_placeholder/storage/immich"])
StackName string // e.g., "immich"
DisplayName string // e.g., "Immich"
CurrentHDDPath string // e.g., "/mnt/hdd_placeholder"
TargetPath string // e.g., "/mnt/hdd_1"
HDDMounts []string // host-side paths to rsync (e.g., ["/mnt/hdd_placeholder/storage/immich"])
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// MigrateProgress tracks migration state.
@@ -82,6 +84,14 @@ func MigrateAppData(
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] MigrateAppData: "+format, args...)
}
}
dbg("starting migration: stack=%s from=%s to=%s mounts=%d", req.StackName, req.CurrentHDDPath, req.TargetPath, len(req.HDDMounts))
// --- Step 1: Validate ---
if req.CurrentHDDPath == "" {
return fail("validating", "A jelenlegi tárhely nem megadott", fmt.Errorf("empty current HDD path"))
@@ -107,8 +117,11 @@ func MigrateAppData(
}
}
dbg("estimated total size: %s (%d bytes)", bytesHuman(totalBytes), totalBytes)
// Check free space on target
freeBytes := getFreeBytes(req.TargetPath)
dbg("target free space: %s (%d bytes)", bytesHuman(freeBytes), freeBytes)
if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes {
return fail("validating", fmt.Sprintf(
"Nincs elég szabad hely a céltárolón: szükséges ~%s, szabad %s",
@@ -126,15 +139,18 @@ func MigrateAppData(
send("stopping", "Alkalmazás leállítva", 10, 0, totalBytes)
// --- Step 3: rsync ---
dbg("starting rsync phase: %d mount(s) to copy", len(req.HDDMounts))
var bytesCopied int64
for i, srcPath := range req.HDDMounts {
// Determine destination path: replace CurrentHDDPath prefix with TargetPath.
// H13: Require trailing separator to prevent /mnt/hdd matching /mnt/hdd_backup/data.
if srcPath != req.CurrentHDDPath && !strings.HasPrefix(srcPath, req.CurrentHDDPath+"/") {
dbg("skipping mount %s (not under %s)", srcPath, req.CurrentHDDPath)
continue
}
relPath := strings.TrimPrefix(srcPath, req.CurrentHDDPath)
dstPath := filepath.Join(req.TargetPath, relPath)
dbg("rsync %d/%d: %s → %s", i+1, len(req.HDDMounts), srcPath, dstPath)
// Ensure destination parent exists
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
@@ -162,6 +178,7 @@ func MigrateAppData(
send("updating", "Konfiguráció frissítése...", 75, bytesCopied, totalBytes)
// --- Step 4: Update app.yaml HDD_PATH ---
dbg("updating config: HDD_PATH %s → %s for stack %s", req.CurrentHDDPath, req.TargetPath, req.StackName)
if err := updateFn(req.StackName, req.TargetPath); err != nil {
send("rolling_back", "Konfiguráció frissítése sikertelen, visszaállítás...", 0, bytesCopied, totalBytes)
_ = startFn(req.StackName)
@@ -178,6 +195,7 @@ func MigrateAppData(
return fail("starting", "Alkalmazás indítása sikertelen az új tárolóról", err)
}
dbg("migration completed: stack=%s bytesCopied=%d elapsed=%ds", req.StackName, bytesCopied, int(time.Since(start).Seconds()))
send("done",
fmt.Sprintf("Áthelyezés kész! Az alkalmazás az új tárolóról fut. (Régi adat: %s, idő: %ds)",
req.CurrentHDDPath, int(time.Since(start).Seconds())),
+43 -4
View File
@@ -5,11 +5,14 @@ package storage
import (
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// lsblkOutput matches the top-level JSON from lsblk -J.
@@ -193,7 +196,13 @@ func partitionToParentDisk(devPath string) string {
// enrichWithBlkid fills in missing FSType, UUID, and Label on partitions using blkid.
// Probes each partition individually via /host-dev (Docker overrides /dev with its own
// tmpfs, so the host block devices are accessible at /host-dev instead).
func enrichWithBlkid(disks []BlockDevice) {
func enrichWithBlkid(disks []BlockDevice, logger *log.Logger, debug bool) {
dbg := func(format string, args ...interface{}) {
if debug && logger != nil {
logger.Printf("[DEBUG] enrichWithBlkid: "+format, args...)
}
}
for i := range disks {
for j := range disks[i].Partitions {
p := &disks[i].Partitions[j]
@@ -202,16 +211,25 @@ func enrichWithBlkid(disks []BlockDevice) {
if p.FSType == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "TYPE", hostPath).Output(); err == nil {
p.FSType = strings.TrimSpace(string(out))
dbg("blkid TYPE %s → %q", hostPath, p.FSType)
} else {
dbg("blkid TYPE %s failed: %v", hostPath, err)
}
}
if p.UUID == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "UUID", hostPath).Output(); err == nil {
p.UUID = strings.TrimSpace(string(out))
dbg("blkid UUID %s → %q", hostPath, p.UUID)
} else {
dbg("blkid UUID %s failed: %v", hostPath, err)
}
}
if p.Label == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "LABEL", hostPath).Output(); err == nil {
p.Label = strings.TrimSpace(string(out))
dbg("blkid LABEL %s → %q", hostPath, p.Label)
} else {
dbg("blkid LABEL %s failed: %v", hostPath, err)
}
}
}
@@ -220,7 +238,13 @@ func enrichWithBlkid(disks []BlockDevice) {
// ScanDisks detects all block devices and classifies them into
// available (not mounted, not system) and system/mounted disks.
func ScanDisks() (*ScanResult, error) {
func ScanDisks(logger *log.Logger, debug bool) (*ScanResult, error) {
dbg := func(format string, args ...interface{}) {
if debug && logger != nil {
logger.Printf("[DEBUG] ScanDisks: "+format, args...)
}
}
out, err := exec.Command(
"lsblk", "-J", "-b",
"-o", "NAME,PATH,SIZE,TYPE,FSTYPE,MOUNTPOINT,MODEL,RM",
@@ -229,11 +253,15 @@ func ScanDisks() (*ScanResult, error) {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
dbg("raw lsblk JSON: %s", util.TruncateStr(string(out), 500))
var parsed lsblkOutput
if err := json.Unmarshal(out, &parsed); err != nil {
return nil, fmt.Errorf("lsblk JSON parse failed: %w", err)
}
dbg("lsblk returned %d block devices", len(parsed.BlockDevices))
// Get system disk names from host fstab (works correctly inside container)
systemDiskNames := getSystemDiskNames()
@@ -287,16 +315,27 @@ func ScanDisks() (*ScanResult, error) {
isSystem := systemDiskNames[dev.Name]
bd.Mounted = anyMounted || isSystem
classification := "available"
if isSystem || anyMounted {
classification = "system"
result.SystemDisks = append(result.SystemDisks, bd)
} else {
result.AvailableDisks = append(result.AvailableDisks, bd)
}
dbg("disk %s: model=%q size=%s partitions=%d removable=%v classification=%s",
bd.Name, bd.Model, bd.Size, len(bd.Partitions), bd.Removable, classification)
for _, p := range bd.Partitions {
dbg(" partition %s: fstype=%q mountpoint=%q size=%s", p.Name, p.FSType, p.MountPoint, p.Size)
}
}
dbg("classification result: %d available, %d system", len(result.AvailableDisks), len(result.SystemDisks))
// Enrich FSType, UUID, Label from blkid (lsblk can't probe fstype in container)
enrichWithBlkid(result.AvailableDisks)
enrichWithBlkid(result.SystemDisks)
enrichWithBlkid(result.AvailableDisks, logger, debug)
enrichWithBlkid(result.SystemDisks, logger, debug)
return result, nil
}
+5 -2
View File
@@ -2,9 +2,12 @@
package storage
import "fmt"
import (
"fmt"
"log"
)
// ScanDisks is not supported on non-Linux platforms.
func ScanDisks() (*ScanResult, error) {
func ScanDisks(logger *log.Logger, debug bool) (*ScanResult, error) {
return nil, fmt.Errorf("storage init is only supported on Linux")
}