v0.24.0 — Pre-testing observability: debug logging, diagnostic dump, startup self-test

- Add [DEBUG] logging across all modules (backup, storage, sync, selfupdate, monitor, notify, report, assets, setup) gated behind logging.level: "debug" - Add /api/debug/dump endpoint returning full controller state JSON (debug only) - Add startup self-test validating 9 subsystems (Docker, dirs, storage, hub, restic repos, metrics DB) with pass/warn/fail summary - New packages: internal/selftest, internal/util - Constructor/signature changes: debug bool params, logger params on RunHealthCheck and BuildReport, smart watchdog probe logging Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-21 18:32:26 +01:00
parent 6f02536243
commit be7803c0ac
30 changed files with 1281 additions and 67 deletions
@@ -13,6 +13,7 @@ import (

 	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
 )

 // DBDumper can run a database dump for a specific stack.
@@ -29,12 +30,13 @@ type CrossDriveRunner struct {
 	stacksDir          string // path to stacks dir (for infra backup)
 	controllerYAMLPath string // path to controller.yaml (for infra backup)
 	logger             *log.Logger
+	debug              bool
 	mu                 sync.Mutex
 	running            map[string]bool // per-app running state
 }

 // NewCrossDriveRunner creates a new CrossDriveRunner.
-func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, systemDataPath, stacksDir string, logger *log.Logger) *CrossDriveRunner {
+func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, systemDataPath, stacksDir string, logger *log.Logger, debug bool) *CrossDriveRunner {
 	return &CrossDriveRunner{
 		sett:               sett,
 		stackProvider:      provider,
@@ -42,6 +44,7 @@ func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, sy
 		stacksDir:          stacksDir,
 		controllerYAMLPath: "/opt/docker/felhom-controller/controller.yaml",
 		logger:             logger,
+		debug:              debug,
 		running:            make(map[string]bool),
 	}
 }
@@ -67,6 +70,11 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e
 		return fmt.Errorf("cross-drive backup not configured or disabled for %s", stackName)
 	}

+	if r.debug {
+		r.logger.Printf("[DEBUG] RunAppBackup: starting for %s, dest=%s, schedule=%s, method=%s",
+			stackName, cfg.DestinationPath, cfg.Schedule, cfg.Method)
+	}
+
 	// Prevent concurrent runs for the same app
 	r.mu.Lock()
 	if r.running[stackName] {
@@ -84,12 +92,18 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e
 	// Check if source or destination drive is disconnected
 	srcDrive := r.stackProvider.GetStackHDDPath(stackName)
 	if srcDrive != "" && r.sett.IsDisconnected(srcDrive) {
+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAppBackup: source drive disconnected for %s: %s", stackName, srcDrive)
+		}
 		r.mu.Lock()
 		r.running[stackName] = false
 		r.mu.Unlock()
 		return fmt.Errorf("source drive disconnected: %s", srcDrive)
 	}
 	if r.sett.IsDisconnected(cfg.DestinationPath) {
+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAppBackup: destination drive disconnected for %s: %s", stackName, cfg.DestinationPath)
+		}
 		r.mu.Lock()
 		r.running[stackName] = false
 		r.mu.Unlock()
@@ -107,6 +121,9 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e

 	// Trigger fresh DB dump for this app before cross-drive backup
 	if r.dbDumper != nil {
+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAppBackup: triggering pre-backup DB dump for %s", stackName)
+		}
 		if err := r.dbDumper.DumpStackDB(ctx, stackName); err != nil {
 			r.logger.Printf("[WARN] Pre-backup DB dump failed for %s: %v — proceeding with user data backup", stackName, err)
 			// Non-fatal: user data backup is still valuable without fresh dump
@@ -120,6 +137,9 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e

 	// Resolve HDD mounts for this app (may be empty for config-only apps)
 	mounts := r.stackProvider.GetStackHDDMounts(stackName)
+	if r.debug {
+		r.logger.Printf("[DEBUG] RunAppBackup: %s has %d HDD mount(s): %v", stackName, len(mounts), mounts)
+	}

 	// Safety: destination must not overlap with any source
 	for _, m := range mounts {
@@ -145,6 +165,9 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e
 	destDir := AppSecondaryRsyncPath(cfg.DestinationPath, stackName)
 	if sz, err := dirSizeBytes(destDir); err == nil {
 		sizeHuman = humanizeBytes(sz)
+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAppBackup: %s backup size at destination: %s", stackName, sizeHuman)
+		}
 	}

 	r.logger.Printf("[INFO] Cross-drive backup completed: %s (%s)", stackName, duration.Round(time.Second))
@@ -155,6 +178,10 @@ func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) e
 // RunAllScheduled runs cross-drive backups for all apps matching the schedule.
 // Runs sequentially (disk I/O bound).
 func (r *CrossDriveRunner) RunAllScheduled(ctx context.Context, schedule string) error {
+	if r.debug {
+		r.logger.Printf("[DEBUG] RunAllScheduled: starting for schedule=%s", schedule)
+	}
+
 	// Auto-enable Tier 2 for small apps (no HDD mounts) before running backups
 	r.AutoEnableSmallApps()

@@ -163,18 +190,39 @@ func (r *CrossDriveRunner) RunAllScheduled(ctx context.Context, schedule string)

 	configs := r.sett.GetAllCrossDriveConfigs()
 	if len(configs) == 0 {
+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAllScheduled: no cross-drive configs found")
+		}
 		return nil
 	}

+	if r.debug {
+		r.logger.Printf("[DEBUG] RunAllScheduled: %d total cross-drive config(s) found", len(configs))
+	}
+
 	var errs []string
+	var scheduled, skippedDisabled, skippedWrongSchedule int
 	for stackName, cfg := range configs {
 		if !cfg.Enabled {
+			if r.debug {
+				r.logger.Printf("[DEBUG] RunAllScheduled: skipping %s — disabled", stackName)
+			}
+			skippedDisabled++
 			continue
 		}
 		if cfg.Schedule != schedule {
+			if r.debug {
+				r.logger.Printf("[DEBUG] RunAllScheduled: skipping %s — schedule mismatch (has=%s, want=%s)", stackName, cfg.Schedule, schedule)
+			}
+			skippedWrongSchedule++
 			continue
 		}

+		if r.debug {
+			r.logger.Printf("[DEBUG] RunAllScheduled: queuing %s for backup (dest=%s)", stackName, cfg.DestinationPath)
+		}
+		scheduled++
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -186,6 +234,11 @@ func (r *CrossDriveRunner) RunAllScheduled(ctx context.Context, schedule string)
 		}
 	}

+	if r.debug {
+		r.logger.Printf("[DEBUG] RunAllScheduled: done — %d scheduled, %d disabled, %d wrong schedule, %d errors",
+			scheduled, skippedDisabled, skippedWrongSchedule, len(errs))
+	}
+
 	if len(errs) > 0 {
 		return fmt.Errorf("cross-drive backup errors: %s", strings.Join(errs, "; "))
 	}
@@ -216,6 +269,9 @@ func (r *CrossDriveRunner) AnyRunning() bool {
 // (≥10 GB free, <90% used) to protect OS stability; external drives just need
 // ≥100 MB. Non-mount-point destinations are allowed with a logged warning.
 func (r *CrossDriveRunner) ValidateDestination(path string) error {
+	if r.debug {
+		r.logger.Printf("[DEBUG] ValidateDestination: checking path=%s", path)
+	}
 	if path == "" {
 		return fmt.Errorf("destination path is empty")
 	}
@@ -226,6 +282,9 @@ func (r *CrossDriveRunner) ValidateDestination(path string) error {
 		return fmt.Errorf("destination %s does not exist", path)
 	}
 	onSystemDrive := !system.IsMountPoint(path)
+	if r.debug {
+		r.logger.Printf("[DEBUG] ValidateDestination: path=%s, isMountPoint=%v", path, !onSystemDrive)
+	}
 	if onSystemDrive {
 		r.logger.Printf("[WARN] Destination %s is not a separate mount point (system drive) — backup will proceed but data is not protected against drive failure", path)
 	}
@@ -237,6 +296,10 @@ func (r *CrossDriveRunner) ValidateDestination(path string) error {
 		r.logger.Printf("[WARN] Cannot determine disk usage for %s — proceeding without space verification", path)
 		return nil
 	}
+	if r.debug {
+		r.logger.Printf("[DEBUG] ValidateDestination: path=%s, availGB=%.1f, usedPct=%.0f%%, onSystemDrive=%v",
+			path, di.AvailGB, di.UsedPercent, onSystemDrive)
+	}
 	if onSystemDrive {
 		// System drive: protect OS stability — require ≥10 GB free and <90% used
 		if di.AvailGB < 10 {
@@ -251,6 +314,9 @@ func (r *CrossDriveRunner) ValidateDestination(path string) error {
 			return fmt.Errorf("destination %s has insufficient free space (%.1f GB free)", path, di.AvailGB)
 		}
 	}
+	if r.debug {
+		r.logger.Printf("[DEBUG] ValidateDestination: path=%s passed all checks", path)
+	}
 	return nil
 }

@@ -258,6 +324,9 @@ func (r *CrossDriveRunner) ValidateDestination(path string) error {

 func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBase string, mounts []string) error {
 	destDir := AppSecondaryRsyncPath(destBase, stackName)
+	if r.debug {
+		r.logger.Printf("[DEBUG] runRsyncBackup: stack=%s, destBase=%s, destDir=%s, %d mount(s)", stackName, destBase, destDir, len(mounts))
+	}
 	if err := os.MkdirAll(destDir, 0755); err != nil {
 		return fmt.Errorf("creating rsync dest dir: %w", err)
 	}
@@ -296,9 +365,16 @@ func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBa
 			"--exclude", "backups/*.dump",
 			src, dst)
 		r.logger.Printf("[DEBUG] rsync: %s → %s", src, dst)
-		if out, err := cmd.CombinedOutput(); err != nil {
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			if r.debug {
+				r.logger.Printf("[DEBUG] runRsyncBackup: rsync failed for %s: %s", srcMount, util.TruncateStr(strings.TrimSpace(string(out)), 500))
+			}
 			return fmt.Errorf("rsync failed for %s: %v (%s)", srcMount, err, strings.TrimSpace(string(out)))
 		}
+		if r.debug {
+			r.logger.Printf("[DEBUG] runRsyncBackup: rsync OK for mount %s → %s", src, dst)
+		}
 	}

 	// --- Copy DB dumps for this stack from its home drive ---
@@ -423,20 +499,35 @@ func (r *CrossDriveRunner) syncInfraConfig(ctx context.Context) {
 func (r *CrossDriveRunner) AutoEnableSmallApps() {
 	storagePaths := r.sett.GetStoragePaths()
 	if len(storagePaths) < 2 {
+		if r.debug {
+			r.logger.Printf("[DEBUG] AutoEnableSmallApps: fewer than 2 storage paths (%d) — skipping", len(storagePaths))
+		}
 		return // no secondary drive available
 	}

 	deployed := r.stackProvider.ListDeployedStacks()
 	existingConfigs := r.sett.GetAllCrossDriveConfigs()

+	if r.debug {
+		r.logger.Printf("[DEBUG] AutoEnableSmallApps: %d deployed stacks, %d existing configs, %d storage paths",
+			len(deployed), len(existingConfigs), len(storagePaths))
+	}
+
+	var autoEnabled int
 	for _, stack := range deployed {
 		// Skip if already has cross-drive config (user has touched it)
 		if _, exists := existingConfigs[stack.Name]; exists {
+			if r.debug {
+				r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — already has cross-drive config", stack.Name)
+			}
 			continue
 		}

 		// Skip if app has HDD mounts (large user data — needs manual config)
 		if mounts := r.stackProvider.GetStackHDDMounts(stack.Name); len(mounts) > 0 {
+			if r.debug {
+				r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — has %d HDD mount(s)", stack.Name, len(mounts))
+			}
 			continue
 		}

@@ -450,6 +541,9 @@ func (r *CrossDriveRunner) AutoEnableSmallApps() {
 			}
 		}
 		if destPath == "" {
+			if r.debug {
+				r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — no suitable destination found", stack.Name)
+			}
 			continue // no suitable destination found
 		}

@@ -464,8 +558,13 @@ func (r *CrossDriveRunner) AutoEnableSmallApps() {
 			r.logger.Printf("[WARN] Auto-enable Tier 2 failed for %s: %v", stack.Name, err)
 			continue
 		}
+		autoEnabled++
 		r.logger.Printf("[INFO] Auto-enabled Tier 2 backup for %s → %s (no HDD mounts, daily rsync)", stack.Name, destPath)
 	}
+
+	if r.debug && autoEnabled > 0 {
+		r.logger.Printf("[DEBUG] AutoEnableSmallApps: auto-enabled %d app(s)", autoEnabled)
+	}
 }

 // --- helpers ---