admin/felhom-controller
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

slice 8C Phase B.2 + C.1/C.2: retire disk subsystem + rewire disk mgmt to agent

Browse Source 
Retired (~12.3k LOC): internal/storage/* (scan/format/attach/migrate/safety),
backup restic/crossdrive/restore_drives/disk_layout/local_infra/restore_scan/
paths + restore_app, report/infra_backup*/infra_pull, setup/scanner,
monitor/watchdog+pinger, web/storage_handlers+handler_restore. Surgically split
backup.Manager to app-data only (DB dumps + volume tars + app restore; dropped
restic + cross-drive + snapshot history). Fixed router/main/web wiring.
Added agent-backed disk API (web/agent_disk_handlers.go): /api/disks list/
assign/eject/format proxying agentapi; data-bearing format refusal -> HTTP 409
'operator authorization required'. report/config_pull.go keeps the setup
fresh-install config download. go build + go test green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
authentik Default Admin
2026-06-10 13:57:27 +02:00
parent 0294513906
commit abe4e8e619
47 changed files with 404 additions and 12317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controller/cmd/controller/main.go
+23 -361
View File
@@ -25,8 +25,8 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
@@ -40,7 +40,6 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/setup"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
"gitea.dooplex.hu/admin/felhom-controller/internal/storage"
catalogsync "gitea.dooplex.hu/admin/felhom-controller/internal/sync"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
"gitea.dooplex.hu/admin/felhom-controller/internal/web"
@@ -186,40 +185,22 @@ func main() {
logger.Println("[INFO] Metrics collector started (60s interval)")
}
// --- Initialize health pinger (legacy, will be removed) ---
pinger := monitor.NewPinger(&cfg.Monitoring, logger)
pinger.SetDebug(cfg.Logging.Level == "debug")
// Deprecation notice for ping UUIDs
// Deprecation notice for ping UUIDs (Healthchecks pinging retired — the Hub
// now owns monitoring; disk-tier backup moved to the host agent in slice 8C).
uuids := cfg.Monitoring.PingUUIDs
if uuids.Heartbeat != "" || uuids.SystemHealth != "" || uuids.DBDump != "" || uuids.Backup != "" || uuids.BackupIntegrity != "" {
logger.Println("[INFO] Healthchecks ping UUIDs configured but no longer used — monitoring is now handled by the Hub")
}
// --- Initialize backup manager ---
// --- Initialize backup manager (app-data only: DB dumps + Docker-volume tars) ---
var backupMgr *backup.Manager
stackProv := &stackAdapter{
mgr: stackMgr,
getStoragePaths: func() []settings.StoragePath { return sett.GetStoragePaths() },
}
if cfg.Backup.Enabled {
backupMgr = backup.NewManager(cfg, pinger, sett, logger)
backupMgr = backup.NewManager(cfg, sett, logger)
backupMgr.SetStackProvider(stackProv)
backupMgr.AfterBackup = func() {
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
backupMgr.RefreshCache(nextDBDump, nextBackup)
}
go backupMgr.LoadSnapshotHistory()
}
// --- Initialize cross-drive backup runner ---
crossDriveRunner := backup.NewCrossDriveRunner(sett, stackProv, cfg.Paths.SystemDataPath, cfg.Paths.StacksDir, logger, cfg.Logging.Level == "debug")
// Wire cross-drive → backup manager for pre-backup DB dumps
if backupMgr != nil {
crossDriveRunner.SetDBDumper(backupMgr)
crossDriveRunner.SetVolumeDumper(backupMgr)
}
// --- Initialize alert manager ---
@@ -259,26 +240,14 @@ func main() {
return stackMgr.RunHealthProbes()
})
// Heartbeat — lightweight "I'm alive" signal
sched.Every("heartbeat", 5*time.Minute, func(ctx context.Context) error {
pinger.Ping(cfg.Monitoring.PingUUIDs.Heartbeat, "")
return nil
})
// System health ping
// System health check — refreshes dashboard alerts and notifies on changes.
// Healthchecks.io pinging has been retired (the Hub now owns monitoring).
healthInterval, err := time.ParseDuration(cfg.Monitoring.SystemHealthInterval)
if err != nil {
healthInterval = 5 * time.Minute
}
sched.Every("system-health", healthInterval, func(ctx context.Context) error {
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), logger)
body := healthReport.FormatMessage()
healthUUID := cfg.Monitoring.PingUUIDs.SystemHealth
if healthReport.Status == "fail" {
pinger.Fail(healthUUID, body)
} else {
pinger.Ping(healthUUID, body)
}
// Refresh dashboard alerts from health report
updateAvailable := false
latestVersion := ""
@@ -322,6 +291,8 @@ func main() {
// Backup daily jobs
if cfg.Backup.Enabled && backupMgr != nil {
// App-data backup: daily database dumps. Disk-tier (restic snapshots,
// cross-drive, integrity check, infra backup) has moved to the host agent.
sched.Daily("db-dump", cfg.Backup.DBDumpSchedule, func(ctx context.Context) error {
err := backupMgr.RunDBDumps(ctx)
if err != nil {
@@ -331,53 +302,11 @@ func main() {
}
return err
})
sched.Daily("backup", cfg.Backup.ResticSchedule, func(ctx context.Context) error {
err := backupMgr.RunBackup(ctx)
if err != nil {
notifier.NotifyBackupFailed("Biztonsági mentés sikertelen", err.Error())
} else {
notifier.NotifyBackupCompleted(notify.BackupDetails{})
}
// Phase 3: Chain cross-drive backups immediately after restic (regardless of restic success)
// Daily jobs run every night; weekly jobs only on Sunday
if crossDriveRunner != nil {
if cdErr := crossDriveRunner.RunAllScheduled(ctx, "daily"); cdErr != nil {
logger.Printf("[WARN] Cross-drive daily backup had errors: %v", cdErr)
}
if time.Now().Weekday() == time.Sunday {
if cdErr := crossDriveRunner.RunAllScheduled(ctx, "weekly"); cdErr != nil {
logger.Printf("[WARN] Cross-drive weekly backup had errors: %v", cdErr)
}
}
}
// Push infra backup to Hub after all backup tiers complete
if hubPusher != nil && cfg.Hub.Enabled {
go pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
}
// Write local infra backup to all connected drives
go writeLocalInfraBackup(cfg, sett, stackProv, logger)
return err
})
// Weekly integrity check — Sunday 04:00
sched.Daily("backup-integrity", "04:00", func(ctx context.Context) error {
if time.Now().Weekday() != time.Sunday {
return nil
}
err := backupMgr.RunIntegrityCheck(ctx)
if err != nil {
notifier.NotifyIntegrityFailed("Mentés integritás ellenőrzés sikertelen", err.Error())
} else {
notifier.NotifyIntegrityOK("Mentés integritás ellenőrzés sikeres")
}
return err
})
// Cache refresh: every 5 minutes
sched.Every("backup-cache", 5*time.Minute, func(ctx context.Context) error {
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
backupMgr.RefreshCache(nextDBDump, nextBackup)
backupMgr.RefreshCache(nextDBDump)
return nil
})
}
@@ -451,35 +380,8 @@ func main() {
}
}
// --- Storage watchdog ---
storageWatchdog := monitor.NewStorageWatchdog(sett, &watchdogStackAdapter{mgr: stackMgr}, notifier, cfg, logger)
storageWatchdog.SetAlertRefresh(func() {
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), logger)
updateAvailable := false
latestVersion := ""
if updater != nil {
status := updater.GetStatus()
if status.LastCheck != nil {
updateAvailable = status.LastCheck.UpdateAvailable
latestVersion = status.LastCheck.LatestVersion
}
}
alertMgr.Refresh(healthReport, cfg, backupMgr, updateAvailable, latestVersion, sett.GetStoragePaths())
})
if hubPusher != nil {
storageWatchdog.SetHubReportPusher(func() {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), logger)
hubPusher.Push(r)
})
}
if backupMgr != nil {
storageWatchdog.SetRepoUnlocker(func(ctx context.Context, repoPath string) error {
return backupMgr.UnlockRepo(ctx, repoPath)
})
}
sched.Every("storage-watchdog", 5*time.Second, func(ctx context.Context) error {
return storageWatchdog.Check(ctx)
})
// Storage watchdog (disk disconnect/reconnect detection) has moved to the host
// agent (slice 8C) — the controller no longer owns disk-level monitoring.
// --- Asset syncer (download from Hub) ---
var assetsSyncer *assets.Syncer
@@ -531,21 +433,6 @@ func main() {
"selftest_fail": selfTestResult.Fail,
})
// Heartbeat ping
pinger.Ping(cfg.Monitoring.PingUUIDs.Heartbeat, "startup")
logger.Println("[INFO] Startup heartbeat ping sent")
// System health ping
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), logger)
body := healthReport.FormatMessage()
healthUUID := cfg.Monitoring.PingUUIDs.SystemHealth
if healthReport.Status == "fail" {
pinger.Fail(healthUUID, body)
} else {
pinger.Ping(healthUUID, body)
}
logger.Printf("[INFO] Startup health ping sent (status: %s)", healthReport.Status)
// Hub report
if hubPusher != nil {
if cfg.Hub.Enabled {
@@ -565,10 +452,6 @@ func main() {
if pushErr != nil {
logger.Printf("[WARN] Startup hub report failed after 3 attempts — next scheduled push in %s", cfg.Hub.PushInterval)
}
// Also push infra backup on startup
go pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
// Write local infra backup to all connected drives
go writeLocalInfraBackup(cfg, sett, stackProv, logger)
} else {
// Send a minimal "disabled" notification so hub knows reporting is intentionally off
r := &report.Report{
@@ -602,8 +485,7 @@ func main() {
if cfg.Backup.Enabled && backupMgr != nil {
go func() {
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
backupMgr.RefreshCache(nextDBDump, nextBackup)
backupMgr.RefreshCache(nextDBDump)
}()
}
@@ -626,14 +508,11 @@ func main() {
}()
// --- Initialize API router ---
apiRouter := api.NewRouter(cfg, *configPath, sett, stackMgr, syncer, cpuCollector, backupMgr, crossDriveRunner, metricsStore, updater, notifier, logger)
apiRouter := api.NewRouter(cfg, *configPath, sett, stackMgr, syncer, cpuCollector, backupMgr, metricsStore, updater, notifier, logger)
if hubPusher != nil {
apiRouter.OnConfigApplied = func() {
pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
}
apiRouter.OnCrossDriveComplete = func() {
pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
writeLocalInfraBackup(cfg, sett, stackProv, logger)
// Infra backup push is now the host agent's responsibility; the controller
// only refreshes the Hub report after a config apply.
}
}
if assetsSyncer != nil {
@@ -694,11 +573,10 @@ func main() {
apiRouter.SetDebug(cfg.Logging.Level == "debug")
// --- Initialize web server ---
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, crossDriveRunner, sched, sett, alertMgr, notifier, updater, logger, Version)
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, sched, sett, alertMgr, notifier, updater, logger, Version)
webServer.SetEncryptionKey(encKey)
webServer.SetAppExporter(appExporter)
webServer.SetIntegrationManager(integrationMgr)
webServer.SetStorageWatchdog(storageWatchdog)
if assetsSyncer != nil {
webServer.SetAssetsSyncer(assetsSyncer)
}
@@ -726,14 +604,6 @@ func main() {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), logger)
return hubPusher.Push(r)
}
dc.TriggerHubInfraPush = func() error {
pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
return nil
}
}
dc.TriggerLocalInfraWrite = func() error {
writeLocalInfraBackup(cfg, sett, stackProv, logger)
return nil
}
dc.HubConnectivityTest = func() (int, int64, error) {
start := time.Now()
@@ -765,55 +635,18 @@ func main() {
webServer.SetDebugCallbacks(dc)
}
// --- Initialize drive migrator ---
driveMigrator := &storage.DriveMigrator{
Sett: sett,
StackProvider: &driveMigrateStackAdapter{mgr: stackMgr},
Logger: logger,
}
// Only set BackupTrigger if backup is enabled (avoid non-nil interface with nil concrete value)
if backupMgr != nil {
driveMigrator.BackupTrigger = backupMgr
}
driveMigrator.AlertRefresh = func() {
healthReport := monitor.RunHealthCheck(cfg, cpuCollector, sett.GetStoragePaths(), logger)
updateAvailable := false
latestVersion := ""
if updater != nil {
status := updater.GetStatus()
if status.LastCheck != nil {
updateAvailable = status.LastCheck.UpdateAvailable
latestVersion = status.LastCheck.LatestVersion
}
}
alertMgr.Refresh(healthReport, cfg, backupMgr, updateAvailable, latestVersion, sett.GetStoragePaths())
}
if hubPusher != nil {
driveMigrator.PushHubReport = func() {
r := report.BuildReport(cfg, *configPath, stackMgr, backupMgr, cpuCollector, metricsStore, Version, sett.GetStoragePaths(), sett.GetGeoRestriction(), logger)
hubPusher.Push(r)
}
driveMigrator.PushInfraBackup = func() {
pushInfraBackup(cfg, sett, stackProv, hubPusher, logger)
}
}
driveMigrator.SyncFBMounts = func() {
webServer.SyncFileBrowserMounts()
}
webServer.SetDriveMigrator(driveMigrator)
// Wire migration-active check into backup manager
if backupMgr != nil {
backupMgr.MigrationActiveCheck = driveMigrator.IsActive
}
// Drive migration (full-drive move) has moved to the host agent (slice 8C);
// the controller no longer runs a DriveMigrator.
// --- Build HTTP mux ---
mux := http.NewServeMux()
// API routes (no auth for health endpoint, auth for everything else)
mux.HandleFunc("/api/health", apiRouter.HealthHandler)
// Storage API routes handled by web server (longer prefix takes precedence over /api/)
mux.Handle("/api/storage/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeStorageAPI))))
// Disk management API — thin proxy to the host agent (slice 8C). The agent owns
// disk execution; the controller forwards list/assign/eject/format.
mux.Handle("/api/disks", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
mux.Handle("/api/disks/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeDiskAPI))))
// App export/import API routes handled by web server
mux.Handle("/api/export/", webServer.RequireAuth(webServer.CsrfProtect(http.HandlerFunc(webServer.ServeExportAPI))))
// Debug API routes handled by web server (debug-mode gating inside handler)
@@ -1028,102 +861,6 @@ func (a *geoStackAdapter) GetDeployedHostnames() map[string]string {
return result
}
// watchdogStackAdapter implements monitor.WatchdogStackProvider using stacks.Manager.
type watchdogStackAdapter struct {
mgr *stacks.Manager
}
func (a *watchdogStackAdapter) ListDeployedStacks() []monitor.WatchdogStackInfo {
var result []monitor.WatchdogStackInfo
for _, s := range a.mgr.GetStacks() {
if !s.Deployed {
continue
}
result = append(result, monitor.WatchdogStackInfo{Name: s.Name})
}
return result
}
func (a *watchdogStackAdapter) GetStackHDDPath(name string) string {
s, ok := a.mgr.GetStack(name)
if !ok {
return ""
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
return filepath.Clean(appCfg.Env["HDD_PATH"])
}
return ""
}
func (a *watchdogStackAdapter) StopStack(name string) error {
return a.mgr.StopStack(name)
}
func (a *watchdogStackAdapter) StartStack(name string) error {
return a.mgr.StartStack(name)
}
// driveMigrateStackAdapter implements storage.StackProviderForMigration using stacks.Manager.
type driveMigrateStackAdapter struct {
mgr *stacks.Manager
}
func (a *driveMigrateStackAdapter) ListDeployedStacks() []storage.StackSummaryForMigration {
var result []storage.StackSummaryForMigration
for _, s := range a.mgr.GetStacks() {
if !s.Deployed {
continue
}
result = append(result, storage.StackSummaryForMigration{
Name: s.Name,
DisplayName: s.Meta.DisplayName,
})
}
return result
}
func (a *driveMigrateStackAdapter) GetStackHDDPath(name string) string {
s, ok := a.mgr.GetStack(name)
if !ok {
return ""
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg != nil && appCfg.Env["HDD_PATH"] != "" {
return filepath.Clean(appCfg.Env["HDD_PATH"])
}
return ""
}
func (a *driveMigrateStackAdapter) StopStack(name string) error {
return a.mgr.StopStack(name)
}
func (a *driveMigrateStackAdapter) StartStack(name string) error {
return a.mgr.StartStack(name)
}
func (a *driveMigrateStackAdapter) UpdateStackHDDPath(name, newPath string) error {
s, ok := a.mgr.GetStack(name)
if !ok {
return fmt.Errorf("stack not found: %s", name)
}
stackDir := filepath.Dir(s.ComposePath)
appCfg := stacks.LoadAppConfig(stackDir)
if appCfg == nil {
return fmt.Errorf("app.yaml not found for stack: %s", name)
}
appCfg.Env["HDD_PATH"] = newPath
return stacks.SaveAppConfig(stackDir, appCfg, nil, nil)
}
func (a *driveMigrateStackAdapter) StackExists(name string) bool {
_, ok := a.mgr.GetStack(name)
return ok
}
// exportAdapter implements appexport.ExportStackProvider using stacks.Manager.
type exportAdapter struct {
mgr *stacks.Manager
@@ -1271,36 +1008,6 @@ func (a *exportAdapter) RemoveStackVolumes(name string) error {
return nil
}
// pushInfraBackup builds and sends the infrastructure snapshot to the Hub.
func pushInfraBackup(cfg *config.Config, sett *settings.Settings,
stackProv *stackAdapter, pusher *report.Pusher, logger *log.Logger) {
encKeyPath := filepath.Join(cfg.Paths.DataDir, "encryption.key")
ib, err := report.BuildInfraBackup(
cfg.Customer.ID, cfg.Customer.Domain, Version,
"/opt/docker/felhom-controller/controller.yaml",
filepath.Join(cfg.Paths.DataDir, "settings.json"),
cfg.Backup.ResticPasswordFile,
encKeyPath,
cfg.Paths.SystemDataPath,
sett, stackProv, logger,
)
if err != nil {
logger.Printf("[WARN] Failed to build infra backup: %v", err)
return
}
data, err := json.Marshal(ib)
if err != nil {
logger.Printf("[WARN] Failed to marshal infra backup: %v", err)
return
}
if err := pusher.PushInfraBackup(data); err != nil {
logger.Printf("[WARN] Failed to push infra backup to Hub: %v", err)
}
}
// fileExists returns true if the path exists (file or directory).
func fileExists(path string) bool {
_, err := os.Stat(path)
@@ -1461,51 +1168,6 @@ func runSetupMode(cfg *config.Config, logger *log.Logger) {
logger.Println("[INFO] Setup wizard stopped")
}
// writeLocalInfraBackup builds an infra snapshot and writes it to all connected drives.
func writeLocalInfraBackup(cfg *config.Config, sett *settings.Settings,
stackProv *stackAdapter, logger *log.Logger) {
encKeyPath := filepath.Join(cfg.Paths.DataDir, "encryption.key")
ib, err := report.BuildInfraBackup(
cfg.Customer.ID, cfg.Customer.Domain, Version,
"/opt/docker/felhom-controller/controller.yaml",
filepath.Join(cfg.Paths.DataDir, "settings.json"),
cfg.Backup.ResticPasswordFile,
encKeyPath,
cfg.Paths.SystemDataPath,
sett, stackProv, logger,
)
if err != nil {
logger.Printf("[WARN] Failed to build infra backup for local write: %v", err)
return
}
data, err := json.Marshal(ib)
if err != nil {
logger.Printf("[WARN] Failed to marshal infra backup for local write: %v", err)
return
}
// Collect all connected drive paths (skip disconnected and decommissioned)
var drives []string
for _, sp := range sett.GetStoragePaths() {
if !sp.Disconnected && !sp.Decommissioned {
drives = append(drives, sp.Path)
}
}
// Also include system data path if set
if cfg.Paths.SystemDataPath != "" {
drives = append(drives, cfg.Paths.SystemDataPath)
}
if len(drives) == 0 {
logger.Println("[DEBUG] No connected drives for local infra backup")
return
}
backup.WriteLocalInfraBackup(data, cfg.Customer.ID, Version, ib.Timestamp, drives, logger, cfg.Logging.Level == "debug")
}
// discoverHDDPaths scans deployed apps' app.yaml for HDD_PATH env values.
func discoverHDDPaths(stacksDir string, logger *log.Logger) []string {
entries, err := os.ReadDir(stacksDir)
controller/internal/api/router.go
+26 -273
View File
@@ -14,10 +14,10 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/assets"
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
"gitea.dooplex.hu/admin/felhom-controller/internal/selfupdate"
@@ -29,25 +29,21 @@ import (
// Router handles all /api/* requests.
type Router struct {
cfg *config.Config
configPath string
sett *settings.Settings
stackMgr *stacks.Manager
syncer *catalogsync.Syncer
cpuCollector *system.CPUCollector
backupMgr *backup.Manager
crossDriveRunner *backup.CrossDriveRunner
metricsStore *metrics.MetricsStore
updater *selfupdate.Updater
notifier *notify.Notifier
logger *log.Logger
cfg *config.Config
configPath string
sett *settings.Settings
stackMgr *stacks.Manager
syncer *catalogsync.Syncer
cpuCollector *system.CPUCollector
backupMgr *backup.Manager
metricsStore *metrics.MetricsStore
updater *selfupdate.Updater
notifier *notify.Notifier
logger *log.Logger
// OnConfigApplied is called after a successful config apply (e.g., to push infra backup).
OnConfigApplied func()
// OnCrossDriveComplete is called after a manual cross-drive backup completes (to push infra backup to Hub).
OnCrossDriveComplete func()
// OnGeoRelevantChange is called after deploy/remove to re-sync geo rules.
OnGeoRelevantChange func()
@@ -89,8 +85,8 @@ func (r *Router) SetIntegrationManager(im *integrations.Manager) {
r.integrationMgr = im
}
func NewRouter(cfg *config.Config, configPath string, sett *settings.Settings, stackMgr *stacks.Manager, syncer *catalogsync.Syncer, cpuCollector *system.CPUCollector, backupMgr *backup.Manager, crossDrive *backup.CrossDriveRunner, metricsStore *metrics.MetricsStore, updater *selfupdate.Updater, notif *notify.Notifier, logger *log.Logger) *Router {
return &Router{cfg: cfg, configPath: configPath, sett: sett, stackMgr: stackMgr, syncer: syncer, cpuCollector: cpuCollector, backupMgr: backupMgr, crossDriveRunner: crossDrive, metricsStore: metricsStore, updater: updater, notifier: notif, logger: logger}
func NewRouter(cfg *config.Config, configPath string, sett *settings.Settings, stackMgr *stacks.Manager, syncer *catalogsync.Syncer, cpuCollector *system.CPUCollector, backupMgr *backup.Manager, metricsStore *metrics.MetricsStore, updater *selfupdate.Updater, notif *notify.Notifier, logger *log.Logger) *Router {
return &Router{cfg: cfg, configPath: configPath, sett: sett, stackMgr: stackMgr, syncer: syncer, cpuCollector: cpuCollector, backupMgr: backupMgr, metricsStore: metricsStore, updater: updater, notifier: notif, logger: logger}
}
type apiResponse struct {
@@ -209,22 +205,6 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case strings.HasPrefix(path, "/stacks/") && req.Method == http.MethodDelete && !hasSubpath(path, "/stacks/"):
r.deleteStack(w, req, trimSegment(path, "/stacks/"))
// POST /api/stacks/{name}/cross-backup — save cross-drive config
case hasSuffix(path, "/cross-backup") && req.Method == http.MethodPost && !hasSuffix(path, "/cross-backup/run") && !hasSuffix(path, "/cross-backup/status"):
r.saveCrossBackupConfig(w, req, extractName(path, "/cross-backup"))
// POST /api/stacks/{name}/cross-backup/run — trigger manual run
case hasSuffix(path, "/cross-backup/run") && req.Method == http.MethodPost:
r.triggerCrossBackup(w, req, extractName(path, "/cross-backup/run"))
// GET /api/stacks/{name}/cross-backup/status — poll status
case hasSuffix(path, "/cross-backup/status") && req.Method == http.MethodGet:
r.getCrossBackupStatus(w, req, extractName(path, "/cross-backup/status"))
// POST /api/backup/cross-drive/run-all — trigger all scheduled cross-drive backups
case path == "/backup/cross-drive/run-all" && req.Method == http.MethodPost:
r.triggerAllCrossBackups(w, req)
// POST /api/sync — trigger immediate catalog sync
case path == "/sync" && req.Method == http.MethodPost:
r.triggerSync(w, req)
@@ -241,10 +221,6 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case path == "/backup/run" && req.Method == http.MethodPost:
r.triggerBackup(w, req)
// GET /api/backup/snapshots
case path == "/backup/snapshots" && req.Method == http.MethodGet:
r.backupSnapshots(w, req)
// GET /api/metrics/system
case path == "/metrics/system" && req.Method == http.MethodGet:
r.metricsSystem(w, req)
@@ -587,8 +563,8 @@ func (r *Router) getStackBackupData(w http.ResponseWriter, _ *http.Request, name
// Compute the drive path for this stack (HDD or system data path)
var drivePath string
if r.crossDriveRunner != nil {
drivePath = r.crossDriveRunner.GetAppDrivePath(name)
if r.backupMgr != nil {
drivePath = r.backupMgr.GetAppDrivePath(name)
}
resp, err := r.stackMgr.GetStackBackupData(name, drivePath)
@@ -618,15 +594,13 @@ func (r *Router) removeStack(w http.ResponseWriter, req *http.Request, name stri
}
r.dbg("removeStack: name=%s removeHDDData=%v removeBackups=%v", name, body.RemoveHDDData, body.RemoveBackups)
// Compute backup paths to remove if requested
// Compute backup paths to remove if requested. Disk-tier (cross-drive rsync)
// backup has moved to the host agent; only the app-data DB-dump path is removed here.
var backupPaths []string
if body.RemoveBackups && r.crossDriveRunner != nil {
drivePath := r.crossDriveRunner.GetAppDrivePath(name)
if body.RemoveBackups && r.backupMgr != nil {
drivePath := r.backupMgr.GetAppDrivePath(name)
if drivePath != "" {
backupPaths = append(backupPaths,
backup.AppDBDumpPath(drivePath, name),
backup.AppSecondaryRsyncPath(drivePath, name),
)
backupPaths = append(backupPaths, backup.AppDBDumpPath(drivePath, name))
}
}
@@ -734,7 +708,7 @@ func (r *Router) backupStatus(w http.ResponseWriter, _ *http.Request) {
return
}
dbDump, backupSt := r.backupMgr.GetStatus()
dbDump := r.backupMgr.GetStatus()
data := map[string]interface{}{
"enabled": true,
"running": r.backupMgr.IsRunning(),
@@ -749,27 +723,11 @@ func (r *Router) backupStatus(w http.ResponseWriter, _ *http.Request) {
}
}
if backupSt != nil {
backupData := map[string]interface{}{
"last_run": backupSt.LastRun,
"success": backupSt.Success,
"duration": backupSt.Duration.String(),
}
if backupSt.Snapshot != nil {
backupData["snapshot_id"] = backupSt.Snapshot.SnapshotID
backupData["files_new"] = backupSt.Snapshot.FilesNew
backupData["data_added"] = backupSt.Snapshot.DataAdded
}
if backupSt.RepoStats != nil {
backupData["repo_size"] = backupSt.RepoStats.TotalSize
backupData["snapshot_count"] = backupSt.RepoStats.SnapshotCount
}
data["backup"] = backupData
}
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: data})
}
// triggerBackup runs the app-data database dumps. Disk-tier (restic) backup has
// moved to the host agent (slice 8C).
func (r *Router) triggerBackup(w http.ResponseWriter, _ *http.Request) {
r.dbg("triggerBackup: backupMgr=%v", r.backupMgr != nil)
if r.backupMgr == nil {
@@ -783,82 +741,12 @@ func (r *Router) triggerBackup(w http.ResponseWriter, _ *http.Request) {
return
}
r.logger.Println("[INFO] [api] Manual backup triggered")
go r.backupMgr.RunFullBackup(context.Background())
r.logger.Println("[INFO] [api] Manual app-data backup (DB dump) triggered")
go r.backupMgr.RunDBDumps(context.Background())
writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: "Mentés elindítva"})
}
func (r *Router) backupSnapshots(w http.ResponseWriter, req *http.Request) {
if r.backupMgr == nil {
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: []interface{}{}})
return
}
stackName := req.URL.Query().Get("stack")
var snapshots []backup.SnapshotInfo
var err error
if stackName != "" {
// Per-app: only snapshots from the app's home drive
snapshots, err = r.backupMgr.ListSnapshotsForApp(stackName, 20)
} else {
// Fallback: all snapshots (general use)
snapshots, err = r.backupMgr.ListAllSnapshots(50)
}
if err != nil {
r.logger.Printf("[ERROR] [api] Failed to list backup snapshots: %v", err)
writeJSON(w, http.StatusInternalServerError, apiResponse{OK: false, Error: err.Error()})
return
}
// Enrich snapshots with drive labels from storage paths
if r.sett != nil {
storagePaths := r.sett.GetStoragePaths()
for i := range snapshots {
repoPath := snapshots[i].RepoPath
for _, sp := range storagePaths {
if strings.HasPrefix(repoPath, sp.Path) {
snapshots[i].DriveLabel = sp.Label
break
}
}
}
// Append Tier 2 (cross-drive rsync) entry if available for this app
if stackName != "" {
cdCfg := r.sett.GetCrossDriveConfig(stackName)
if cdCfg != nil && cdCfg.Enabled && cdCfg.LastStatus == "ok" && cdCfg.LastRun != "" {
lastRun, _ := time.Parse(time.RFC3339, cdCfg.LastRun)
if !lastRun.IsZero() {
// Resolve drive label for destination
var destLabel string
for _, sp := range storagePaths {
if sp.Path == cdCfg.DestinationPath {
destLabel = sp.Label
break
}
}
tier2 := backup.SnapshotInfo{
ID: "tier2-rsync",
Time: lastRun,
Tier: 2,
Source: "rsync",
DriveLabel: destLabel,
}
snapshots = append(snapshots, tier2)
}
}
}
}
if snapshots == nil {
snapshots = []backup.SnapshotInfo{}
}
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: snapshots})
}
// --- Metrics handlers ---
func (r *Router) metricsSystem(w http.ResponseWriter, req *http.Request) {
@@ -955,141 +843,6 @@ func (r *Router) metricsSysInfo(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: info})
}
// --- Cross-drive backup handlers ---
func (r *Router) saveCrossBackupConfig(w http.ResponseWriter, req *http.Request, name string) {
if r.crossDriveRunner == nil {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "cross-drive runner not available"})
return
}
limitBody(w, req)
var body struct {
Enabled bool `json:"enabled"`
DestinationPath string `json:"destination_path"`
Schedule string `json:"schedule"`
}
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "invalid request body"})
return
}
// Validate schedule
if body.Schedule != "daily" && body.Schedule != "weekly" && body.Schedule != "manual" {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "schedule must be 'daily', 'weekly', or 'manual'"})
return
}
// C9: Validate DestinationPath against registered storage paths to prevent path traversal.
if body.Enabled && body.DestinationPath != "" {
registeredPaths := r.sett.GetStoragePaths()
validDest := false
for _, sp := range registeredPaths {
if body.DestinationPath == sp.Path {
validDest = true
break
}
}
if !validDest {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "destination_path must be a registered storage path"})
return
}
}
// Preserve existing runtime status
existing := r.sett.GetCrossDriveConfig(name)
var lastRun, lastStatus, lastError, lastDuration, lastSize string
if existing != nil {
lastRun, lastStatus, lastError, lastDuration, lastSize =
existing.LastRun, existing.LastStatus, existing.LastError, existing.LastDuration, existing.LastSizeHuman
}
cfg := &settings.CrossDriveBackup{
Enabled: body.Enabled,
Method: "rsync",
DestinationPath: body.DestinationPath,
Schedule: body.Schedule,
LastRun: lastRun,
LastStatus: lastStatus,
LastError: lastError,
LastDuration: lastDuration,
LastSizeHuman: lastSize,
}
if err := r.sett.SetCrossDriveConfig(name, cfg); err != nil {
r.logger.Printf("[ERROR] [api] Failed to save cross-drive config for %s: %v", name, err)
writeJSON(w, http.StatusInternalServerError, apiResponse{OK: false, Error: err.Error()})
return
}
r.logger.Printf("[INFO] [api] Cross-drive backup config saved for %s: dest=%s schedule=%s",
name, body.DestinationPath, body.Schedule)
writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: "Cross-drive backup configuration saved"})
}
func (r *Router) triggerCrossBackup(w http.ResponseWriter, req *http.Request, name string) {
if r.crossDriveRunner == nil {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "cross-drive runner not available"})
return
}
if r.crossDriveRunner.IsRunning(name) {
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: "Mentés már folyamatban"})
return
}
r.logger.Printf("[INFO] [api] Cross-drive backup triggered for: %s", name)
go func() {
if err := r.crossDriveRunner.RunAppBackup(context.Background(), name); err != nil {
r.logger.Printf("[ERROR] [api] Cross-drive backup failed for %s: %v", name, err)
}
if r.OnCrossDriveComplete != nil {
r.OnCrossDriveComplete()
}
}()
writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: "Mentés elindítva"})
}
func (r *Router) getCrossBackupStatus(w http.ResponseWriter, _ *http.Request, name string) {
cfg := r.sett.GetCrossDriveConfig(name)
if cfg == nil {
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: map[string]interface{}{"configured": false}})
return
}
writeJSON(w, http.StatusOK, apiResponse{OK: true, Data: map[string]interface{}{
"configured": true,
"enabled": cfg.Enabled,
"method": "rsync",
"schedule": cfg.Schedule,
"running": r.crossDriveRunner != nil && r.crossDriveRunner.IsRunning(name),
"last_run": cfg.LastRun,
"last_status": cfg.LastStatus,
"last_error": cfg.LastError,
"last_duration": cfg.LastDuration,
"last_size": cfg.LastSizeHuman,
}})
}
func (r *Router) triggerAllCrossBackups(w http.ResponseWriter, _ *http.Request) {
if r.crossDriveRunner == nil {
writeJSON(w, http.StatusBadRequest, apiResponse{OK: false, Error: "cross-drive runner not available"})
return
}
r.logger.Println("[INFO] [api] All cross-drive backups triggered")
go func() {
ctx := context.Background()
if err := r.crossDriveRunner.RunAllScheduled(ctx, "daily"); err != nil {
r.logger.Printf("[ERROR] [api] Cross-drive run-all error: %v", err)
}
if err := r.crossDriveRunner.RunAllScheduled(ctx, "weekly"); err != nil {
r.logger.Printf("[ERROR] [api] Cross-drive run-all weekly error: %v", err)
}
if r.OnCrossDriveComplete != nil {
r.OnCrossDriveComplete()
}
}()
writeJSON(w, http.StatusOK, apiResponse{OK: true, Message: "Összes mentés elindítva"})
}
// parseTimeRange reads range or from/to query params.
func parseTimeRange(req *http.Request) (from, to time.Time) {
to = time.Now()
controller/internal/backup/backup.go
+24 -928
View File
File diff suppressed because it is too large Load Diff
controller/internal/backup/crossdrive.go
-734
View File
@@ -1,734 +0,0 @@
package backup
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// DBDumper can run a database dump for a specific stack.
type DBDumper interface {
DumpStackDB(ctx context.Context, stackName string) error
}
// VolumeDumper can dump Docker named volumes for a specific stack.
type VolumeDumper interface {
DumpAppVolumes(stackName string) error
DumpAppVolumesSafe(stackName string) error // stops stack before dump, restarts after
}
// CrossDriveRunner handles per-app backup to secondary storage.
type CrossDriveRunner struct {
sett *settings.Settings
stackProvider StackDataProvider
dbDumper DBDumper
volDumper VolumeDumper
systemDataPath string // fallback drive for SSD-only apps
stacksDir string // path to stacks dir (for infra backup)
controllerYAMLPath string // path to controller.yaml (for infra backup)
logger *log.Logger
debug bool
mu sync.Mutex
running map[string]bool // per-app running state
}
// NewCrossDriveRunner creates a new CrossDriveRunner.
func NewCrossDriveRunner(sett *settings.Settings, provider StackDataProvider, systemDataPath, stacksDir string, logger *log.Logger, debug bool) *CrossDriveRunner {
return &CrossDriveRunner{
sett: sett,
stackProvider: provider,
systemDataPath: systemDataPath,
stacksDir: stacksDir,
controllerYAMLPath: "/opt/docker/felhom-controller/controller.yaml",
logger: logger,
debug: debug,
running: make(map[string]bool),
}
}
// SetDBDumper sets the DB dumper for pre-backup database dumps.
// Called after backup manager is initialized (avoids circular init dependency).
func (r *CrossDriveRunner) SetDBDumper(d DBDumper) {
r.dbDumper = d
}
// SetVolumeDumper sets the volume dumper for pre-backup Docker volume dumps.
func (r *CrossDriveRunner) SetVolumeDumper(d VolumeDumper) {
r.volDumper = d
}
// GetAppDrivePath returns the drive path for an app (HDD path or system data path fallback).
func (r *CrossDriveRunner) GetAppDrivePath(stackName string) string {
if hddPath := r.stackProvider.GetStackHDDPath(stackName); hddPath != "" {
return hddPath
}
return r.systemDataPath
}
// RunAppBackup runs cross-drive backup for a single app.
func (r *CrossDriveRunner) RunAppBackup(ctx context.Context, stackName string) error {
cfg := r.sett.GetCrossDriveConfig(stackName)
if cfg == nil || !cfg.Enabled {
return fmt.Errorf("cross-drive backup not configured or disabled for %s", stackName)
}
if r.debug {
r.logger.Printf("[DEBUG] RunAppBackup: starting for %s, dest=%s, schedule=%s, method=%s",
stackName, cfg.DestinationPath, cfg.Schedule, cfg.Method)
}
// Prevent concurrent runs for the same app
r.mu.Lock()
if r.running[stackName] {
r.mu.Unlock()
return fmt.Errorf("cross-drive backup already running for %s", stackName)
}
r.running[stackName] = true
r.mu.Unlock()
defer func() {
r.mu.Lock()
r.running[stackName] = false
r.mu.Unlock()
}()
// Check if source or destination drive is disconnected — skip silently (not an error)
srcDrive := r.stackProvider.GetStackHDDPath(stackName)
if srcDrive != "" && r.sett.IsDisconnected(srcDrive) {
r.logger.Printf("[WARN] [backup] Cross-drive backup skipped for %s: source drive disconnected (%s)", stackName, srcDrive)
return nil
}
if r.sett.IsDisconnected(cfg.DestinationPath) {
r.logger.Printf("[WARN] [backup] Cross-drive backup skipped for %s: destination drive disconnected (%s)", stackName, cfg.DestinationPath)
return nil
}
if !r.sett.IsStoragePathKnown(cfg.DestinationPath) {
r.logger.Printf("[WARN] [backup] Cross-drive backup skipped for %s: destination not a registered storage (%s)", stackName, cfg.DestinationPath)
return nil
}
if !r.sett.IsStoragePathSchedulable(cfg.DestinationPath) {
r.logger.Printf("[WARN] [backup] Cross-drive backup skipped for %s: destination drive inactive (%s)", stackName, cfg.DestinationPath)
return nil
}
// Mark as running in settings
_ = r.sett.UpdateCrossDriveStatus(stackName, func(c *settings.CrossDriveBackup) {
c.LastStatus = "running"
})
start := time.Now()
r.logger.Printf("[INFO] [backup] Cross-drive backup starting: %s → %s (rsync)",
stackName, cfg.DestinationPath)
// Trigger fresh DB dump for this app before cross-drive backup
if r.dbDumper != nil {
if r.debug {
r.logger.Printf("[DEBUG] RunAppBackup: triggering pre-backup DB dump for %s", stackName)
}
if err := r.dbDumper.DumpStackDB(ctx, stackName); err != nil {
r.logger.Printf("[WARN] [backup] Pre-backup DB dump failed for %s: %v — proceeding with user data backup", stackName, err)
}
}
// Trigger fresh volume dump for this app before cross-drive backup
if r.volDumper != nil {
if r.debug {
r.logger.Printf("[DEBUG] RunAppBackup: triggering pre-backup volume dump for %s", stackName)
}
if err := r.volDumper.DumpAppVolumesSafe(stackName); err != nil {
r.logger.Printf("[WARN] [backup] Pre-backup volume dump failed for %s: %v — proceeding with backup", stackName, err)
}
}
if err := r.ValidateDestination(cfg.DestinationPath); err != nil {
r.updateStatus(stackName, "error", err.Error(), time.Since(start), "")
return fmt.Errorf("destination validation failed: %w", err)
}
// Resolve HDD mounts for this app (may be empty for config-only apps)
mounts := r.stackProvider.GetStackHDDMounts(stackName)
if r.debug {
r.logger.Printf("[DEBUG] RunAppBackup: %s has %d HDD mount(s): %v", stackName, len(mounts), mounts)
}
// Safety: destination must not overlap with any source
for _, m := range mounts {
if system.PathsOverlap(cfg.DestinationPath, m) {
msg := fmt.Sprintf("destination %s overlaps with source %s — aborted", cfg.DestinationPath, m)
r.updateStatus(stackName, "error", msg, time.Since(start), "")
return fmt.Errorf("%s", msg)
}
}
runErr := r.runRsyncBackup(ctx, stackName, cfg.DestinationPath, mounts)
duration := time.Since(start)
if runErr != nil {
r.logger.Printf("[ERROR] [backup] Cross-drive backup failed: %s: %v", stackName, runErr)
r.updateStatus(stackName, "error", runErr.Error(), duration, "")
return runErr
}
// Calculate backup size
var sizeHuman string
destDir := AppSecondaryRsyncPath(cfg.DestinationPath, stackName)
if sz, err := dirSizeBytes(destDir); err == nil {
sizeHuman = humanizeBytes(sz)
if r.debug {
r.logger.Printf("[DEBUG] RunAppBackup: %s backup size at destination: %s", stackName, sizeHuman)
}
}
r.logger.Printf("[INFO] [backup] Cross-drive backup completed: %s (%s)", stackName, duration.Round(time.Second))
r.updateStatus(stackName, "ok", "", duration, sizeHuman)
return nil
}
// RunAllScheduled runs cross-drive backups for all apps matching the schedule.
// Runs sequentially (disk I/O bound).
func (r *CrossDriveRunner) RunAllScheduled(ctx context.Context, schedule string) error {
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: starting for schedule=%s", schedule)
}
// Auto-enable Tier 2 for small apps (no HDD mounts) before running backups
r.AutoEnableSmallApps()
// Sync infrastructure config to all secondary destinations
r.syncInfraConfig(ctx)
configs := r.sett.GetAllCrossDriveConfigs()
if len(configs) == 0 {
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: no cross-drive configs found")
}
return nil
}
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: %d total cross-drive config(s) found", len(configs))
}
var errs []string
var scheduled, skippedDisabled, skippedWrongSchedule int
r.logger.Printf("[INFO] [backup] Cross-drive backup: starting scheduled run for %d configured app(s), schedule=%s", len(configs), schedule)
for stackName, cfg := range configs {
if !cfg.Enabled {
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: skipping %s — disabled", stackName)
}
skippedDisabled++
continue
}
if cfg.Schedule != schedule {
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: skipping %s — schedule mismatch (has=%s, want=%s)", stackName, cfg.Schedule, schedule)
}
skippedWrongSchedule++
continue
}
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: queuing %s for backup (dest=%s)", stackName, cfg.DestinationPath)
}
scheduled++
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err := r.RunAppBackup(ctx, stackName); err != nil {
errs = append(errs, fmt.Sprintf("%s: %v", stackName, err))
}
}
if r.debug {
r.logger.Printf("[DEBUG] RunAllScheduled: done — %d scheduled, %d disabled, %d wrong schedule, %d errors",
scheduled, skippedDisabled, skippedWrongSchedule, len(errs))
}
r.logger.Printf("[INFO] [backup] Cross-drive backup complete: %d succeeded, %d failed", scheduled-len(errs), len(errs))
if len(errs) > 0 {
return fmt.Errorf("cross-drive backup errors: %s", strings.Join(errs, "; "))
}
return nil
}
// RunAllConfigured runs cross-drive backup for all enabled apps, ignoring schedule.
// Used by the debug page to trigger all backups regardless of their configured schedule.
func (r *CrossDriveRunner) RunAllConfigured(ctx context.Context) error {
if r.debug {
r.logger.Printf("[DEBUG] RunAllConfigured: starting for all enabled apps")
}
r.AutoEnableSmallApps()
r.syncInfraConfig(ctx)
configs := r.sett.GetAllCrossDriveConfigs()
if len(configs) == 0 {
return nil
}
var errs []string
var ran int
r.logger.Printf("[INFO] [backup] Cross-drive backup: starting all configured app(s), %d total", len(configs))
for stackName, cfg := range configs {
if !cfg.Enabled {
continue
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
ran++
if err := r.RunAppBackup(ctx, stackName); err != nil {
errs = append(errs, fmt.Sprintf("%s: %v", stackName, err))
}
}
if r.debug {
r.logger.Printf("[DEBUG] RunAllConfigured: done — %d ran, %d errors", ran, len(errs))
}
r.logger.Printf("[INFO] [backup] Cross-drive backup complete: %d succeeded, %d failed", ran-len(errs), len(errs))
if len(errs) > 0 {
return fmt.Errorf("cross-drive errors: %s", strings.Join(errs, "; "))
}
return nil
}
// IsRunning returns true if the given app's backup is currently running.
func (r *CrossDriveRunner) IsRunning(stackName string) bool {
r.mu.Lock()
defer r.mu.Unlock()
return r.running[stackName]
}
// AnyRunning returns true if any cross-drive backup is currently in progress.
func (r *CrossDriveRunner) AnyRunning() bool {
r.mu.Lock()
defer r.mu.Unlock()
for _, running := range r.running {
if running {
return true
}
}
return false
}
// ValidateDestination checks that the destination path exists, is writable,
// and has sufficient free space. System-drive destinations get stricter limits
// (≥10 GB free, <90% used) to protect OS stability; external drives just need
// ≥100 MB. Non-mount-point destinations are allowed with a logged warning.
func (r *CrossDriveRunner) ValidateDestination(path string) error {
if r.debug {
r.logger.Printf("[DEBUG] ValidateDestination: checking path=%s", path)
}
if path == "" {
return fmt.Errorf("destination path is empty")
}
if r.sett.IsDecommissioned(path) {
return fmt.Errorf("destination %s is decommissioned — choose an active drive", path)
}
if _, err := os.Stat(path); os.IsNotExist(err) {
return fmt.Errorf("destination %s does not exist", path)
}
onSystemDrive := !system.IsMountPoint(path)
if r.debug {
r.logger.Printf("[DEBUG] ValidateDestination: path=%s, isMountPoint=%v", path, !onSystemDrive)
}
if onSystemDrive {
r.logger.Printf("[WARN] [backup] Destination %s is not a separate mount point (system drive) — backup will proceed but data is not protected against drive failure", path)
}
if !system.IsWritable(path) {
return fmt.Errorf("destination %s is not writable", path)
}
di := system.GetDiskUsage(path)
if di == nil {
r.logger.Printf("[WARN] [backup] Cannot determine disk usage for %s — proceeding without space verification", path)
return nil
}
if r.debug {
r.logger.Printf("[DEBUG] ValidateDestination: path=%s, availGB=%.1f, usedPct=%.0f%%, onSystemDrive=%v",
path, di.AvailGB, di.UsedPercent, onSystemDrive)
}
if onSystemDrive {
// System drive: protect OS stability — require ≥10 GB free and <90% used
if di.AvailGB < 10 {
return fmt.Errorf("destination %s is on the system drive with only %.1f GB free — at least 10 GB required to protect OS stability", path, di.AvailGB)
}
if di.UsedPercent >= 90 {
return fmt.Errorf("destination %s is on the system drive at %.0f%% capacity — maximum 90%% allowed", path, di.UsedPercent)
}
} else {
// External drive: just ensure it's not completely full
if di.AvailGB < 0.1 {
return fmt.Errorf("destination %s has insufficient free space (%.1f GB free)", path, di.AvailGB)
}
}
if r.debug {
r.logger.Printf("[DEBUG] ValidateDestination: path=%s passed all checks", path)
}
return nil
}
// --- rsync ---
func (r *CrossDriveRunner) runRsyncBackup(ctx context.Context, stackName, destBase string, mounts []string) error {
destDir := AppSecondaryRsyncPath(destBase, stackName)
if r.debug {
r.logger.Printf("[DEBUG] runRsyncBackup: stack=%s, destBase=%s, destDir=%s, %d mount(s)", stackName, destBase, destDir, len(mounts))
}
if err := os.MkdirAll(destDir, 0755); err != nil {
return fmt.Errorf("creating rsync dest dir: %w", err)
}
seen := make(map[string]bool)
for _, srcMount := range mounts {
var dstPath string
if len(mounts) == 1 {
// Single mount: rsync directly into the stack folder (no extra nesting)
dstPath = destDir
} else {
// Multiple mounts: use the leaf directory name as subfolder
leaf := filepath.Base(srcMount)
if seen[leaf] {
// Disambiguate duplicate leaf names (e.g. two mounts both named "data")
for j := 2; ; j++ {
candidate := fmt.Sprintf("%s_%d", leaf, j)
if !seen[candidate] {
leaf = candidate
break
}
}
}
seen[leaf] = true
dstPath = filepath.Join(destDir, leaf)
}
if err := os.MkdirAll(dstPath, 0755); err != nil {
return fmt.Errorf("creating rsync destination: %w", err)
}
// Ensure trailing slash on source for rsync semantics (copy contents, not the dir itself)
src := strings.TrimRight(srcMount, "/") + "/"
dst := strings.TrimRight(dstPath, "/") + "/"
// Exclude controller-managed directories (underscore prefix) to prevent --delete from removing
// _db/ and _config/ that were created by previous backup runs.
// Exclude app-internal DB dump files — the controller handles DB backups via pg_dump separately.
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete",
"--exclude", "_*",
"--exclude", "backups/*.sql.gz",
"--exclude", "backups/*.sql",
"--exclude", "backups/*.dump",
src, dst)
if r.debug {
r.logger.Printf("[DEBUG] rsync: %s → %s", src, dst)
}
out, err := cmd.CombinedOutput()
if err != nil {
if r.debug {
r.logger.Printf("[DEBUG] runRsyncBackup: rsync failed for %s: %s", srcMount, util.TruncateStr(strings.TrimSpace(string(out)), 500))
}
r.logger.Printf("[ERROR] [backup] Rsync backup for %s failed: %v", stackName, err)
return fmt.Errorf("rsync failed for %s: %v (%s)", srcMount, err, strings.TrimSpace(string(out)))
}
if r.debug {
r.logger.Printf("[DEBUG] runRsyncBackup: rsync OK for mount %s → %s", src, dst)
}
}
// --- Copy DB dumps for this stack from its home drive ---
dbDestDir := filepath.Join(destDir, "_db")
if err := os.MkdirAll(dbDestDir, 0755); err != nil {
return fmt.Errorf("creating DB dump dest dir: %w", err)
}
if err := r.copyStackDBDumps(stackName, dbDestDir); err != nil {
r.logger.Printf("[WARN] [backup] Cross-drive DB dump copy failed for %s: %v", stackName, err)
// Non-fatal: user data is the primary concern
}
// --- Copy volume dumps for this stack from its home drive ---
volDestDir := filepath.Join(destDir, "_volumes")
if err := os.MkdirAll(volDestDir, 0755); err != nil {
return fmt.Errorf("creating volume dump dest dir: %w", err)
}
if err := r.copyStackVolumeDumps(stackName, volDestDir); err != nil {
r.logger.Printf("[WARN] [backup] Cross-drive volume dump copy failed for %s: %v", stackName, err)
// Non-fatal: user data is the primary concern
}
// --- Rsync app config (compose dir) ---
if composePath, ok := r.stackProvider.GetStackComposePath(stackName); ok {
configSrcDir := filepath.Dir(composePath)
configDestDir := filepath.Join(destDir, "_config")
if err := os.MkdirAll(configDestDir, 0755); err != nil {
return fmt.Errorf("creating config dest dir: %w", err)
}
src := strings.TrimRight(configSrcDir, "/") + "/"
dst := strings.TrimRight(configDestDir, "/") + "/"
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", src, dst)
if r.debug {
r.logger.Printf("[DEBUG] rsync config: %s → %s", src, dst)
}
if out, err := cmd.CombinedOutput(); err != nil {
r.logger.Printf("[WARN] [backup] Cross-drive config rsync failed for %s: %v (%s)", stackName, err, strings.TrimSpace(string(out)))
// Non-fatal
}
}
r.logger.Printf("[INFO] [backup] Rsync backup for %s to %s complete", stackName, destDir)
return nil
}
// copyStackDBDumps copies DB dump files for the given stack from its home drive.
// DB dumps are at <drive>/backups/primary/<stack>/db-dumps/<stack>_<dbtype>.sql.
func (r *CrossDriveRunner) copyStackDBDumps(stackName, destDir string) error {
appDrive := r.GetAppDrivePath(stackName)
dumpDir := AppDBDumpPath(appDrive, stackName)
entries, err := os.ReadDir(dumpDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("reading DB dump dir: %w", err)
}
copied := 0
for _, e := range entries {
if e.IsDir() {
continue
}
src := filepath.Join(dumpDir, e.Name())
dst := filepath.Join(destDir, e.Name())
if err := copyFile(src, dst); err != nil {
return fmt.Errorf("copying %s: %w", e.Name(), err)
}
copied++
}
r.logger.Printf("[INFO] [backup] Copied %d DB dumps for %s", copied, stackName)
return nil
}
// copyStackVolumeDumps copies Docker volume dump tars for the given stack from its home drive.
func (r *CrossDriveRunner) copyStackVolumeDumps(stackName, destDir string) error {
appDrive := r.GetAppDrivePath(stackName)
dumpDir := AppVolumeDumpPath(appDrive, stackName)
entries, err := os.ReadDir(dumpDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("reading volume dump dir: %w", err)
}
copied := 0
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".tar") {
continue
}
src := filepath.Join(dumpDir, e.Name())
dst := filepath.Join(destDir, e.Name())
if err := copyFile(src, dst); err != nil {
return fmt.Errorf("copying %s: %w", e.Name(), err)
}
copied++
}
if copied > 0 {
r.logger.Printf("[INFO] [backup] Copied %d volume dump(s) for %s", copied, stackName)
}
return nil
}
// --- infra backup ---
// syncInfraConfig rsyncs infrastructure config (stacks dir + controller.yaml) to all
// secondary backup destinations. Runs once per RunAllScheduled cycle, before per-app backups.
func (r *CrossDriveRunner) syncInfraConfig(ctx context.Context) {
// Collect unique destination drives from enabled cross-drive configs
destDrives := make(map[string]bool)
for _, cfg := range r.sett.GetAllCrossDriveConfigs() {
if cfg.Enabled && cfg.DestinationPath != "" {
destDrives[cfg.DestinationPath] = true
}
}
if len(destDrives) == 0 {
return
}
for dest := range destDrives {
infraDir := SecondaryInfraPath(dest)
if err := os.MkdirAll(infraDir, 0755); err != nil {
r.logger.Printf("[WARN] [backup] Cannot create infra backup dir %s: %v", infraDir, err)
continue
}
// Rsync stacks dir → _infra/stacks/
stacksDest := filepath.Join(infraDir, "stacks") + "/"
if err := os.MkdirAll(stacksDest, 0755); err == nil {
stacksSrc := strings.TrimRight(r.stacksDir, "/") + "/"
cmd := exec.CommandContext(ctx, "rsync", "-a", "--delete", stacksSrc, stacksDest)
if out, err := cmd.CombinedOutput(); err != nil {
r.logger.Printf("[WARN] [backup] Infra rsync (stacks) failed for %s: %v (%s)", dest, err, strings.TrimSpace(string(out)))
}
}
// Copy controller.yaml → _infra/controller.yaml (atomic via copyFile)
if _, err := os.Stat(r.controllerYAMLPath); err == nil {
yamlDest := filepath.Join(infraDir, "controller.yaml")
if err := copyFile(r.controllerYAMLPath, yamlDest); err != nil {
r.logger.Printf("[WARN] [backup] Cannot copy controller.yaml to %s: %v", yamlDest, err)
}
}
r.logger.Printf("[INFO] [backup] Infrastructure config synced to %s", infraDir)
}
}
// --- auto-enable ---
// AutoEnableSmallApps auto-configures cross-drive backup for apps without HDD user data
// when at least 2 storage paths are registered. Apps with existing cross-drive config
// (even if disabled) are never modified.
func (r *CrossDriveRunner) AutoEnableSmallApps() {
storagePaths := r.sett.GetStoragePaths()
if len(storagePaths) < 2 {
if r.debug {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: fewer than 2 storage paths (%d) — skipping", len(storagePaths))
}
return // no secondary drive available
}
deployed := r.stackProvider.ListDeployedStacks()
existingConfigs := r.sett.GetAllCrossDriveConfigs()
if r.debug {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: %d deployed stacks, %d existing configs, %d storage paths",
len(deployed), len(existingConfigs), len(storagePaths))
}
var autoEnabled int
for _, stack := range deployed {
// Skip if already has cross-drive config (user has touched it)
if _, exists := existingConfigs[stack.Name]; exists {
if r.debug {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — already has cross-drive config", stack.Name)
}
continue
}
// Skip if app has HDD mounts (large user data — needs manual config)
if mounts := r.stackProvider.GetStackHDDMounts(stack.Name); len(mounts) > 0 {
if r.debug {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — has %d HDD mount(s)", stack.Name, len(mounts))
}
continue
}
// Find destination: first active storage path that differs from the app's home drive
appDrive := r.GetAppDrivePath(stack.Name)
var destPath string
for _, sp := range storagePaths {
if sp.Path != appDrive && !sp.Disconnected && !sp.Decommissioned {
destPath = sp.Path
break
}
}
if destPath == "" {
if r.debug {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: skipping %s — no suitable destination found", stack.Name)
}
continue // no suitable destination found
}
// Auto-configure daily rsync
cfg := &settings.CrossDriveBackup{
Enabled: true,
Method: "rsync",
DestinationPath: destPath,
Schedule: "daily",
}
if err := r.sett.SetCrossDriveConfig(stack.Name, cfg); err != nil {
r.logger.Printf("[WARN] [backup] Auto-enable Tier 2 failed for %s: %v", stack.Name, err)
continue
}
autoEnabled++
r.logger.Printf("[INFO] [backup] Auto-enabled Tier 2 backup for %s → %s (no HDD mounts, daily rsync)", stack.Name, destPath)
}
if r.debug && autoEnabled > 0 {
r.logger.Printf("[DEBUG] AutoEnableSmallApps: auto-enabled %d app(s)", autoEnabled)
}
}
// --- helpers ---
func (r *CrossDriveRunner) updateStatus(stackName, status, errMsg string, duration time.Duration, sizeHuman string) {
_ = r.sett.UpdateCrossDriveStatus(stackName, func(c *settings.CrossDriveBackup) {
c.LastRun = time.Now().UTC().Format(time.RFC3339)
c.LastStatus = status
c.LastError = errMsg
c.LastDuration = duration.Round(time.Second).String()
if sizeHuman != "" {
c.LastSizeHuman = sizeHuman
}
})
}
// copyFile copies src to dst using buffered streaming I/O (no full-file memory allocation).
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
tmp := dst + ".tmp"
out, err := os.Create(tmp)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
os.Remove(tmp)
return err
}
if err := out.Close(); err != nil {
os.Remove(tmp)
return err
}
return os.Rename(tmp, dst)
}
// dirSizeBytes returns the total byte size of all files under path.
// H7: Walk errors are now propagated instead of silently swallowed.
func dirSizeBytes(path string) (int64, error) {
var total int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if err != nil {
return err // propagate permission/IO errors
}
if !info.IsDir() {
total += info.Size()
}
return nil
})
return total, err
}
controller/internal/backup/disk_layout.go
-19
View File
@@ -1,19 +0,0 @@
package backup
// DiskLayout holds the fstab-derived mount topology for disaster recovery.
type DiskLayout struct {
Mounts []DiskMount `json:"mounts"`
}
// DiskMount represents a single mount entry from fstab.
type DiskMount struct {
UUID string `json:"uuid"`
Label string `json:"label"`
MountPoint string `json:"mount_point"`
FSType string `json:"fs_type"`
SizeBytes int64 `json:"size_bytes"`
FstabOptions string `json:"fstab_options"`
Role string `json:"role"` // "system_data", "hdd_storage"
BindSubdir string `json:"bind_subdir"` // e.g., "felhom_data"
RawMount string `json:"raw_mount"` // e.g., "/mnt/.felhom-raw/hdd_1"
}
controller/internal/backup/local_infra.go
-368
View File
@@ -1,368 +0,0 @@
package backup
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// MaxSchemaVersion is the highest infra backup schema version this controller can read.
const MaxSchemaVersion = 1
// maxLocalHistory is the number of previous backup versions to keep per drive.
const maxLocalHistory = 5
// InfraMetadata is the lightweight metadata file written alongside backup.json.
type InfraMetadata struct {
SchemaVersion int `json:"schema_version"`
Timestamp string `json:"timestamp"`
CustomerID string `json:"customer_id"`
ControllerVersion string `json:"controller_version"`
Checksum string `json:"checksum"` // SHA256 hex of backup.json
}
// WriteLocalInfraBackup writes the infra backup to .felhom-infra-backup/ on each drive.
// Individual drive failures are logged but not returned — the function is best-effort.
func WriteLocalInfraBackup(backupJSON []byte, customerID, controllerVersion, timestamp string, drives []string, logger *log.Logger, debug bool) {
if len(drives) == 0 {
logger.Printf("[DEBUG] No drives configured for local infra backup")
return
}
if debug {
logger.Printf("[DEBUG] WriteLocalInfraBackup: payload size=%d bytes, %d target drive(s): %v", len(backupJSON), len(drives), drives)
}
// Compute checksum of backup data
hash := sha256.Sum256(backupJSON)
checksum := hex.EncodeToString(hash[:])
meta := InfraMetadata{
SchemaVersion: 1,
Timestamp: timestamp,
CustomerID: customerID,
ControllerVersion: controllerVersion,
Checksum: checksum,
}
metaJSON, err := json.MarshalIndent(meta, "", " ")
if err != nil {
logger.Printf("[ERROR] Local infra backup: failed to marshal metadata: %v", err)
return
}
written := 0
for _, drive := range drives {
dir := InfraBackupDir(drive)
if debug {
logger.Printf("[DEBUG] WriteLocalInfraBackup: writing to drive=%s, dir=%s", drive, dir)
}
if err := writeInfraToDir(dir, backupJSON, metaJSON, logger); err != nil {
logger.Printf("[WARN] Local infra backup: failed to write to %s: %v", drive, err)
continue
}
if debug {
logger.Printf("[DEBUG] WriteLocalInfraBackup: write OK to %s", drive)
}
written++
}
logger.Printf("[INFO] Local infra backup written to %d/%d drive(s)", written, len(drives))
}
// writeInfraToDir rotates the current backup into history/ then writes new backup.json and metadata.json.
func writeInfraToDir(dir string, backupData, metaData []byte, logger *log.Logger) error {
if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("creating dir: %w", err)
}
// Rotate current backup to history (best-effort)
rotateToHistory(dir, logger)
// Write backup.json atomically
backupPath := filepath.Join(dir, "backup.json")
if err := atomicWrite(backupPath, backupData, 0600); err != nil {
return fmt.Errorf("writing backup.json: %w", err)
}
// Write metadata.json atomically
metaPath := filepath.Join(dir, "metadata.json")
if err := atomicWrite(metaPath, metaData, 0600); err != nil {
return fmt.Errorf("writing metadata.json: %w", err)
}
return nil
}
// rotateToHistory moves the current backup.json + metadata.json into history/{timestamp}-*.
func rotateToHistory(dir string, logger *log.Logger) {
metaPath := filepath.Join(dir, "metadata.json")
backupPath := filepath.Join(dir, "backup.json")
// Read current metadata to get timestamp
metaData, err := os.ReadFile(metaPath)
if err != nil {
return // no existing backup to rotate
}
var meta InfraMetadata
if err := json.Unmarshal(metaData, &meta); err != nil {
return
}
// Parse timestamp, fall back to file mtime
ts := sanitizeTimestamp(meta.Timestamp)
if ts == "" {
if fi, err := os.Stat(metaPath); err == nil {
ts = fi.ModTime().UTC().Format("20060102T150405Z")
} else {
ts = time.Now().UTC().Format("20060102T150405Z")
}
}
histDir := filepath.Join(dir, "history")
if err := os.MkdirAll(histDir, 0700); err != nil {
if logger != nil {
logger.Printf("[WARN] Local infra history: cannot create history dir: %v", err)
}
return
}
// Move files
histBackup := filepath.Join(histDir, ts+"-backup.json")
histMeta := filepath.Join(histDir, ts+"-metadata.json")
// Copy rather than rename to avoid cross-device issues
if data, err := os.ReadFile(backupPath); err == nil {
os.WriteFile(histBackup, data, 0600) //nolint:errcheck
}
os.WriteFile(histMeta, metaData, 0600) //nolint:errcheck
// Prune old history entries
pruneLocalHistory(histDir, maxLocalHistory, logger)
}
// pruneLocalHistory keeps at most maxKeep metadata+backup pairs, deleting the oldest.
func pruneLocalHistory(histDir string, maxKeep int, logger *log.Logger) {
entries, err := os.ReadDir(histDir)
if err != nil {
return
}
// Collect unique timestamps (each has -backup.json and -metadata.json)
timestamps := make(map[string]bool)
for _, e := range entries {
name := e.Name()
if strings.HasSuffix(name, "-metadata.json") {
ts := strings.TrimSuffix(name, "-metadata.json")
timestamps[ts] = true
}
}
if len(timestamps) <= maxKeep {
return
}
// Sort timestamps ascending (oldest first)
sorted := make([]string, 0, len(timestamps))
for ts := range timestamps {
sorted = append(sorted, ts)
}
sort.Strings(sorted)
// Delete oldest entries beyond limit
toDelete := len(sorted) - maxKeep
for i := 0; i < toDelete; i++ {
ts := sorted[i]
os.Remove(filepath.Join(histDir, ts+"-backup.json"))
os.Remove(filepath.Join(histDir, ts+"-metadata.json"))
if logger != nil {
logger.Printf("[DEBUG] Local infra history: pruned old version %s", ts)
}
}
if logger != nil && toDelete > 0 {
logger.Printf("[INFO] [backup] Pruning old backup versions: kept %d, removed %d", maxKeep, toDelete)
}
}
// sanitizeTimestamp converts an RFC3339 timestamp to a filename-safe format.
func sanitizeTimestamp(ts string) string {
t, err := time.Parse(time.RFC3339, ts)
if err != nil {
t, err = time.Parse(time.RFC3339Nano, ts)
if err != nil {
return ""
}
}
return t.UTC().Format("20060102T150405Z")
}
// atomicWrite writes data to a .tmp file then renames to the target path.
func atomicWrite(path string, data []byte, perm os.FileMode) error {
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, perm); err != nil {
os.Remove(tmp)
return err
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}
// ReadLocalInfraBackup reads and validates an infra backup from a mount point.
// Returns the raw backup JSON, metadata, and any error.
func ReadLocalInfraBackup(mountPath string) ([]byte, *InfraMetadata, error) {
dir := InfraBackupDir(mountPath)
return readInfraBackupFromDir(dir)
}
// ReadLocalInfraBackupFromHistory reads a specific historical version by its timestamp prefix.
func ReadLocalInfraBackupFromHistory(mountPath, historyPrefix string) ([]byte, *InfraMetadata, error) {
histDir := InfraBackupHistoryDir(mountPath)
metaPath := filepath.Join(histDir, historyPrefix+"-metadata.json")
backupPath := filepath.Join(histDir, historyPrefix+"-backup.json")
return readInfraBackupFromFiles(backupPath, metaPath)
}
// LocalBackupVersion holds summary info for a historical backup version found on a drive.
type LocalBackupVersion struct {
Timestamp string `json:"timestamp"`
CustomerID string `json:"customer_id"`
ControllerVersion string `json:"controller_version"`
IntegrityOK bool `json:"integrity_ok"`
Error string `json:"error,omitempty"`
StackCount int `json:"stack_count"`
StackNames []string `json:"stack_names,omitempty"`
DiskCount int `json:"disk_count"`
HistoryFile string `json:"history_file,omitempty"` // empty = current, timestamp prefix for history
}
// ReadLocalInfraHistory reads all historical backup versions from a mount point's history/ directory.
// Returns newest-first. Does NOT include the current backup (use ReadLocalInfraBackup for that).
func ReadLocalInfraHistory(mountPath string) []LocalBackupVersion {
histDir := InfraBackupHistoryDir(mountPath)
entries, err := os.ReadDir(histDir)
if err != nil {
return nil
}
// Collect unique timestamps
var timestamps []string
seen := make(map[string]bool)
for _, e := range entries {
name := e.Name()
if strings.HasSuffix(name, "-metadata.json") {
ts := strings.TrimSuffix(name, "-metadata.json")
if !seen[ts] {
seen[ts] = true
timestamps = append(timestamps, ts)
}
}
}
// Sort descending (newest first)
sort.Sort(sort.Reverse(sort.StringSlice(timestamps)))
var versions []LocalBackupVersion
for _, ts := range timestamps {
v := LocalBackupVersion{HistoryFile: ts}
backupPath := filepath.Join(histDir, ts+"-backup.json")
metaPath := filepath.Join(histDir, ts+"-metadata.json")
backupData, meta, err := readInfraBackupFromFiles(backupPath, metaPath)
if meta != nil {
v.Timestamp = meta.Timestamp
v.CustomerID = meta.CustomerID
v.ControllerVersion = meta.ControllerVersion
}
if err != nil {
v.IntegrityOK = false
v.Error = err.Error()
} else {
v.IntegrityOK = true
ParseBackupCounts(backupData, &v.StackCount, &v.StackNames, &v.DiskCount)
}
versions = append(versions, v)
}
return versions
}
// ParseBackupCounts extracts stack/disk counts from backup JSON (for display purposes).
func ParseBackupCounts(backupJSON []byte, stackCount *int, stackNames *[]string, diskCount *int) {
var parsed struct {
DeployedStacks []struct {
Name string `json:"name"`
DisplayName string `json:"display_name"`
} `json:"deployed_stacks"`
DiskLayout struct {
Mounts []json.RawMessage `json:"mounts"`
} `json:"disk_layout"`
}
if err := json.Unmarshal(backupJSON, &parsed); err != nil {
return
}
*stackCount = len(parsed.DeployedStacks)
*diskCount = len(parsed.DiskLayout.Mounts)
if stackNames != nil {
for _, s := range parsed.DeployedStacks {
name := s.DisplayName
if name == "" {
name = s.Name
}
*stackNames = append(*stackNames, name)
}
}
}
func readInfraBackupFromDir(dir string) ([]byte, *InfraMetadata, error) {
metaPath := filepath.Join(dir, "metadata.json")
backupPath := filepath.Join(dir, "backup.json")
return readInfraBackupFromFiles(backupPath, metaPath)
}
func readInfraBackupFromFiles(backupPath, metaPath string) ([]byte, *InfraMetadata, error) {
// Read metadata
metaData, err := os.ReadFile(metaPath)
if err != nil {
return nil, nil, fmt.Errorf("reading metadata.json: %w", err)
}
var meta InfraMetadata
if err := json.Unmarshal(metaData, &meta); err != nil {
return nil, nil, fmt.Errorf("parsing metadata.json: %w", err)
}
// Check schema version
if meta.SchemaVersion > MaxSchemaVersion {
return nil, &meta, fmt.Errorf("backup schema version %d is newer than supported version %d — upgrade the controller", meta.SchemaVersion, MaxSchemaVersion)
}
// Read backup data
backupData, err := os.ReadFile(backupPath)
if err != nil {
return nil, &meta, fmt.Errorf("reading backup.json: %w", err)
}
// Verify checksum
hash := sha256.Sum256(backupData)
actual := hex.EncodeToString(hash[:])
if actual != meta.Checksum {
return nil, &meta, fmt.Errorf("checksum mismatch: expected %s, got %s", meta.Checksum, actual)
}
return backupData, &meta, nil
}
controller/internal/backup/local_infra_test.go
-163
View File
@@ -1,163 +0,0 @@
package backup
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"log"
"os"
"path/filepath"
"testing"
)
func TestWriteAndReadLocalInfraBackup(t *testing.T) {
tmpDir := t.TempDir()
drive := filepath.Join(tmpDir, "mnt", "hdd_0")
if err := os.MkdirAll(drive, 0755); err != nil {
t.Fatal(err)
}
backupJSON := []byte(`{"customer_id":"test-123","domain":"test.hu","controller_version":"v0.21.0","timestamp":"2026-02-21T10:00:00Z"}`)
logger := testLogger(t)
WriteLocalInfraBackup(backupJSON, "test-123", "v0.21.0", "2026-02-21T10:00:00Z", []string{drive}, logger, false)
// Verify files exist
dir := InfraBackupDir(drive)
if _, err := os.Stat(filepath.Join(dir, "backup.json")); err != nil {
t.Fatalf("backup.json not found: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "metadata.json")); err != nil {
t.Fatalf("metadata.json not found: %v", err)
}
// Read back
data, meta, err := ReadLocalInfraBackup(drive)
if err != nil {
t.Fatalf("ReadLocalInfraBackup failed: %v", err)
}
if string(data) != string(backupJSON) {
t.Errorf("backup data mismatch: got %s", string(data))
}
if meta.SchemaVersion != 1 {
t.Errorf("expected schema version 1, got %d", meta.SchemaVersion)
}
if meta.CustomerID != "test-123" {
t.Errorf("expected customer_id test-123, got %s", meta.CustomerID)
}
if meta.ControllerVersion != "v0.21.0" {
t.Errorf("expected controller version v0.21.0, got %s", meta.ControllerVersion)
}
// Verify checksum
hash := sha256.Sum256(backupJSON)
expected := hex.EncodeToString(hash[:])
if meta.Checksum != expected {
t.Errorf("checksum mismatch: expected %s, got %s", expected, meta.Checksum)
}
}
func TestReadLocalInfraBackup_ChecksumMismatch(t *testing.T) {
tmpDir := t.TempDir()
drive := filepath.Join(tmpDir, "mnt", "hdd_0")
dir := InfraBackupDir(drive)
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
// Write valid metadata with wrong checksum
meta := InfraMetadata{SchemaVersion: 1, Checksum: "0000000000000000000000000000000000000000000000000000000000000000"}
metaJSON, _ := json.Marshal(meta)
os.WriteFile(filepath.Join(dir, "metadata.json"), metaJSON, 0600)
os.WriteFile(filepath.Join(dir, "backup.json"), []byte(`{"test":true}`), 0600)
_, _, err := ReadLocalInfraBackup(drive)
if err == nil {
t.Fatal("expected checksum mismatch error")
}
if got := err.Error(); !contains(got, "checksum mismatch") {
t.Errorf("expected checksum mismatch error, got: %s", got)
}
}
func TestReadLocalInfraBackup_SchemaVersionTooNew(t *testing.T) {
tmpDir := t.TempDir()
drive := filepath.Join(tmpDir, "mnt", "hdd_0")
dir := InfraBackupDir(drive)
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
meta := InfraMetadata{SchemaVersion: 999}
metaJSON, _ := json.Marshal(meta)
os.WriteFile(filepath.Join(dir, "metadata.json"), metaJSON, 0600)
os.WriteFile(filepath.Join(dir, "backup.json"), []byte(`{}`), 0600)
_, _, err := ReadLocalInfraBackup(drive)
if err == nil {
t.Fatal("expected schema version error")
}
if got := err.Error(); !contains(got, "newer than supported") {
t.Errorf("expected schema version error, got: %s", got)
}
}
func TestReadLocalInfraBackup_MissingFiles(t *testing.T) {
tmpDir := t.TempDir()
_, _, err := ReadLocalInfraBackup(tmpDir)
if err == nil {
t.Fatal("expected error for missing files")
}
}
func TestWriteLocalInfraBackup_MultipleDrives(t *testing.T) {
tmpDir := t.TempDir()
drives := []string{
filepath.Join(tmpDir, "drive1"),
filepath.Join(tmpDir, "drive2"),
filepath.Join(tmpDir, "drive3_fail"), // won't be created as a dir, but MkdirAll should handle it
}
for _, d := range drives {
os.MkdirAll(d, 0755)
}
backupJSON := []byte(`{"test":"multi"}`)
logger := testLogger(t)
WriteLocalInfraBackup(backupJSON, "multi-test", "v1.0", "2026-01-01T00:00:00Z", drives, logger, false)
// All 3 should succeed
for _, d := range drives {
data, _, err := ReadLocalInfraBackup(d)
if err != nil {
t.Errorf("drive %s: read failed: %v", d, err)
continue
}
if string(data) != string(backupJSON) {
t.Errorf("drive %s: data mismatch", d)
}
}
}
func TestWriteLocalInfraBackup_NoDrives(t *testing.T) {
logger := testLogger(t)
// Should not panic
WriteLocalInfraBackup([]byte(`{}`), "test", "v1.0", "2026-01-01T00:00:00Z", nil, logger, false)
}
func contains(s, substr string) bool {
return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsStr(s, substr))
}
func containsStr(s, substr string) bool {
for i := 0; i+len(substr) <= len(s); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}
func testLogger(t *testing.T) *log.Logger {
return log.New(os.Stderr, "[test] ", log.LstdFlags)
}
controller/internal/backup/paths.go
-42
View File
@@ -1,42 +0,0 @@
package backup
import "path/filepath"
// Keep-side path helpers (FelhomDataDir, PrimaryBackupPath, AppDBDumpPath,
// AppVolumeDumpPath, AppDataDir) now live in internal/appbackup and are
// re-exposed here via aliases/forwarders in appbackup_bridge.go.
// PrimaryResticRepoPath returns the restic repo path on a drive's primary backup.
func PrimaryResticRepoPath(drivePath string) string {
return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", "restic")
}
// SecondaryBackupPath returns the root secondary backup directory for a drive.
func SecondaryBackupPath(drivePath string) string {
return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary")
}
// AppSecondaryRsyncPath returns the rsync destination for an app's secondary backup.
func AppSecondaryRsyncPath(drivePath, stackName string) string {
return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary", stackName, "rsync")
}
// SecondaryResticRepoPath returns the restic repo path on a drive's secondary backup.
func SecondaryResticRepoPath(drivePath string) string {
return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary", "restic")
}
// SecondaryInfraPath returns the infrastructure config mirror directory on a drive's secondary backup.
func SecondaryInfraPath(drivePath string) string {
return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary", "_infra")
}
// InfraBackupDir returns the hidden infra backup directory on a drive.
func InfraBackupDir(mountPath string) string {
return filepath.Join(mountPath, ".felhom-infra-backup")
}
// InfraBackupHistoryDir returns the history subdirectory for versioned infra backups on a drive.
func InfraBackupHistoryDir(mountPath string) string {
return filepath.Join(mountPath, ".felhom-infra-backup", "history")
}
controller/internal/backup/restic.go
-497
View File
@@ -1,497 +0,0 @@
package backup
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)
// ResticManager handles restic backup operations.
// All methods accept repoPath as parameter to support per-drive repos.
type ResticManager struct {
passwordFile string
logger *log.Logger
customerID string
cacheDir string
debug bool
}
// SnapshotResult holds the outcome of a restic backup.
type SnapshotResult struct {
SnapshotID string
FilesNew int
FilesChanged int
DataAdded string
Duration time.Duration
}
// SnapshotInfo holds information about a restic snapshot.
type SnapshotInfo struct {
ID string `json:"short_id"`
Time time.Time `json:"time"`
Paths []string `json:"paths"`
Tags []string `json:"tags"`
RepoPath string `json:"-"` // set by caller for multi-repo aggregation
Tier int `json:"tier"` // 1 = primary, 2 = secondary
DriveLabel string `json:"drive_label"` // filled by caller from settings
Source string `json:"source"` // "restic" or "rsync"
}
// RepoStats holds repository statistics.
type RepoStats struct {
TotalSize string
TotalSizeBytes int64
SnapshotCount int
LatestSnapshot *SnapshotInfo
}
// NewResticManager creates a new restic manager.
func NewResticManager(cfg *config.Config, logger *log.Logger) *ResticManager {
return &ResticManager{
passwordFile: cfg.Backup.ResticPasswordFile,
logger: logger,
customerID: cfg.Customer.ID,
cacheDir: filepath.Join(cfg.Paths.DataDir, "restic-cache"),
}
}
// SetDebug enables or disables debug logging.
func (r *ResticManager) SetDebug(debug bool) {
r.debug = debug
}
// EnsureInitialized checks if the restic repo exists and initializes it if not.
// Also auto-generates the password file if missing.
func (r *ResticManager) EnsureInitialized(repoPath string) error {
if r.debug {
r.logger.Printf("[DEBUG] [restic] EnsureInitialized: repoPath=%s, passwordFile=%s", repoPath, r.passwordFile)
}
// Ensure password file exists
if _, err := os.Stat(r.passwordFile); os.IsNotExist(err) {
if err := r.generatePassword(); err != nil {
return fmt.Errorf("generating restic password: %w", err)
}
}
// Ensure cache dir exists
os.MkdirAll(r.cacheDir, 0700)
// Check if repo is already initialized
configPath := filepath.Join(repoPath, "config")
if _, err := os.Stat(configPath); err == nil {
r.logger.Printf("[INFO] [backup] Restic repo already initialized at %s", repoPath)
return nil
}
// Ensure repo directory exists
if err := os.MkdirAll(repoPath, 0700); err != nil {
return fmt.Errorf("creating repo dir: %w", err)
}
// Initialize repo
r.logger.Printf("[INFO] [backup] Initializing restic repository at %s", repoPath)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
cmd := r.command(ctx, repoPath, "init")
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("restic init failed: %v — %s", err, truncate(string(out), 200))
}
r.logger.Printf("[INFO] [backup] Restic repository initialized successfully")
return nil
}
// Snapshot creates a new backup snapshot of the given paths.
func (r *ResticManager) Snapshot(repoPath string, paths []string, tags []string) (*SnapshotResult, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
start := time.Now()
if r.debug {
r.logger.Printf("[DEBUG] [restic] Snapshot: repo=%s, paths=%v, tags=%v", repoPath, paths, tags)
}
args := []string{"backup", "--json"}
for _, tag := range tags {
args = append(args, "--tag", tag)
}
args = append(args, "--host", r.customerID)
// Only include paths that exist
var existingPaths []string
for _, p := range paths {
if _, err := os.Stat(p); err == nil {
existingPaths = append(existingPaths, p)
} else {
r.logger.Printf("[WARN] [backup] Backup path does not exist, skipping: %s", p)
}
}
if len(existingPaths) == 0 {
return nil, fmt.Errorf("no backup paths exist")
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] Snapshot: %d/%d paths exist, backing up: %v", len(existingPaths), len(paths), existingPaths)
}
args = append(args, existingPaths...)
cmd := r.command(ctx, repoPath, args...)
out, err := cmd.Output()
if err != nil {
// Check for stale lock — restic writes lock errors to stderr, not stdout
errStr := string(out)
if exitErr, ok := err.(*exec.ExitError); ok {
errStr += string(exitErr.Stderr)
}
if strings.Contains(errStr, "lock") || strings.Contains(errStr, "locked") {
r.logger.Printf("[WARN] [backup] Restic repo locked — attempting unlock")
unlockCmd := r.command(ctx, repoPath, "unlock")
if unlockErr := unlockCmd.Run(); unlockErr != nil {
r.logger.Printf("[WARN] [backup] Restic unlock failed: %v", unlockErr)
}
// Retry once with a fresh context (H9 fix — original may be nearly expired).
retryCtx, retryCancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer retryCancel()
cmd = r.command(retryCtx, repoPath, args...)
out, err = cmd.Output()
if err != nil {
return nil, fmt.Errorf("restic backup failed after unlock: %v", err)
}
} else {
return nil, fmt.Errorf("restic backup failed: %v", err)
}
}
result := &SnapshotResult{
Duration: time.Since(start),
}
// Parse JSON output — look for the summary line
for _, line := range strings.Split(string(out), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
var msg struct {
MessageType string `json:"message_type"`
FilesNew int `json:"files_new"`
FilesChanged int `json:"files_changed"`
DataAdded int64 `json:"data_added"`
SnapshotID string `json:"snapshot_id"`
}
if err := json.Unmarshal([]byte(line), &msg); err != nil {
continue
}
if msg.MessageType == "summary" {
result.SnapshotID = msg.SnapshotID
result.FilesNew = msg.FilesNew
result.FilesChanged = msg.FilesChanged
result.DataAdded = humanizeBytes(msg.DataAdded)
}
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] Snapshot: completed in %s, snapshotID=%s, filesNew=%d, filesChanged=%d, dataAdded=%s",
result.Duration, result.SnapshotID, result.FilesNew, result.FilesChanged, result.DataAdded)
}
r.logger.Printf("[INFO] [backup] Restic snapshot complete for %s", repoPath)
return result, nil
}
// Prune removes old snapshots according to retention policy.
func (r *ResticManager) Prune(repoPath string, retention config.RetentionConfig) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if r.debug {
r.logger.Printf("[DEBUG] [restic] Prune: repo=%s, keepDaily=%d, keepWeekly=%d, keepMonthly=%d",
repoPath, retention.KeepDaily, retention.KeepWeekly, retention.KeepMonthly)
}
start := time.Now()
args := []string{
"forget",
"--keep-daily", fmt.Sprintf("%d", retention.KeepDaily),
"--keep-weekly", fmt.Sprintf("%d", retention.KeepWeekly),
"--keep-monthly", fmt.Sprintf("%d", retention.KeepMonthly),
"--prune",
}
cmd := r.command(ctx, repoPath, args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("restic forget/prune failed: %v — %s", err, truncate(string(out), 200))
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] Prune: completed in %s, output=%d bytes", time.Since(start), len(out))
}
r.logger.Printf("[INFO] [backup] Restic prune completed for %s", repoPath)
return nil
}
// Check verifies repository integrity.
func (r *ResticManager) Check(repoPath string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if r.debug {
r.logger.Printf("[DEBUG] [restic] Check: repo=%s", repoPath)
}
start := time.Now()
cmd := r.command(ctx, repoPath, "check")
out, err := cmd.CombinedOutput()
if err != nil {
if r.debug {
r.logger.Printf("[DEBUG] [restic] Check: failed after %s, output=%s", time.Since(start), truncate(string(out), 300))
}
return fmt.Errorf("restic check failed: %v — %s", err, truncate(string(out), 200))
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] Check: repo=%s OK, completed in %s", repoPath, time.Since(start))
}
r.logger.Printf("[INFO] [backup] Restic check passed for repo %s", repoPath)
return nil
}
// ListSnapshots returns all snapshots, newest first, limited to N entries.
func (r *ResticManager) ListSnapshots(repoPath string, limit int) ([]SnapshotInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if r.debug {
r.logger.Printf("[DEBUG] [restic] ListSnapshots: repo=%s, limit=%d", repoPath, limit)
}
cmd := r.command(ctx, repoPath, "snapshots", "--json")
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("restic snapshots failed: %v", err)
}
var snapshots []SnapshotInfo
if err := json.Unmarshal(out, &snapshots); err != nil {
return nil, fmt.Errorf("parsing snapshot JSON: %v", err)
}
// Reverse for newest first
for i, j := 0, len(snapshots)-1; i < j; i, j = i+1, j-1 {
snapshots[i], snapshots[j] = snapshots[j], snapshots[i]
}
if limit > 0 && len(snapshots) > limit {
snapshots = snapshots[:limit]
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] ListSnapshots: repo=%s, found %d total snapshots, returning %d",
repoPath, len(snapshots), len(snapshots))
}
return snapshots, nil
}
// LatestSnapshot returns the most recent snapshot info.
func (r *ResticManager) LatestSnapshot(repoPath string) (*SnapshotInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
if r.debug {
r.logger.Printf("[DEBUG] [restic] LatestSnapshot: repo=%s", repoPath)
}
cmd := r.command(ctx, repoPath, "snapshots", "--latest", "1", "--json")
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("restic snapshots failed: %v", err)
}
var snapshots []SnapshotInfo
if err := json.Unmarshal(out, &snapshots); err != nil {
return nil, fmt.Errorf("parsing snapshot JSON: %v", err)
}
if len(snapshots) == 0 {
if r.debug {
r.logger.Printf("[DEBUG] [restic] LatestSnapshot: repo=%s, no snapshots found", repoPath)
}
return nil, nil
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] LatestSnapshot: repo=%s, id=%s, time=%s, paths=%v",
repoPath, snapshots[0].ID, snapshots[0].Time.Format(time.RFC3339), snapshots[0].Paths)
}
return &snapshots[0], nil
}
// Stats returns repository statistics.
func (r *ResticManager) Stats(repoPath string) (*RepoStats, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if r.debug {
r.logger.Printf("[DEBUG] [restic] Stats: repo=%s", repoPath)
}
start := time.Now()
stats := &RepoStats{}
// Get repo size
cmd := r.command(ctx, repoPath, "stats", "--json")
out, err := cmd.Output()
if err == nil {
var raw struct {
TotalSize uint64 `json:"total_size"`
}
if json.Unmarshal(out, &raw) == nil {
stats.TotalSizeBytes = int64(raw.TotalSize)
stats.TotalSize = humanizeBytes(stats.TotalSizeBytes)
}
}
// Count snapshots
cmd = r.command(ctx, repoPath, "snapshots", "--json")
out, err = cmd.Output()
if err == nil {
var snapshots []SnapshotInfo
if json.Unmarshal(out, &snapshots) == nil {
stats.SnapshotCount = len(snapshots)
if len(snapshots) > 0 {
latest := snapshots[len(snapshots)-1]
stats.LatestSnapshot = &latest
}
}
}
if r.debug {
latestID := "none"
if stats.LatestSnapshot != nil {
latestID = stats.LatestSnapshot.ID
}
r.logger.Printf("[DEBUG] [restic] Stats: repo=%s, totalSize=%s, snapshots=%d, latest=%s, took %s",
repoPath, stats.TotalSize, stats.SnapshotCount, latestID, time.Since(start))
}
return stats, nil
}
// GetPassword reads and returns the restic repository password.
func (r *ResticManager) GetPassword() (string, error) {
data, err := os.ReadFile(r.passwordFile)
if err != nil {
return "", fmt.Errorf("reading restic password: %w", err)
}
return strings.TrimSpace(string(data)), nil
}
// RestoreAppData restores specific paths from a restic snapshot.
func (r *ResticManager) RestoreAppData(repoPath string, snapshotID string, paths []string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
args := []string{
"restore", snapshotID,
"--target", "/",
}
for _, p := range paths {
args = append(args, "--include", p)
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] RestoreAppData: repo=%s, snapshot=%s, %d include paths=%v",
repoPath, snapshotID, len(paths), paths)
}
start := time.Now()
r.logger.Printf("[INFO] [backup] Restore started: repo=%s, snapshot=%s, paths=%v", repoPath, snapshotID, paths)
cmd := r.command(ctx, repoPath, args...)
output, err := cmd.CombinedOutput()
if err != nil {
r.logger.Printf("[ERROR] [backup] Restore failed: %v, output: %s", err, truncate(string(output), 500))
return fmt.Errorf("restic restore failed: %w", err)
}
if r.debug {
r.logger.Printf("[DEBUG] [restic] RestoreAppData: completed in %s, output=%d bytes", time.Since(start), len(output))
}
r.logger.Printf("[INFO] [backup] Restore completed: snapshot=%s, paths=%v", snapshotID, paths)
return nil
}
// RepoExists checks if a restic repo is initialized at the given path.
func (r *ResticManager) RepoExists(repoPath string) bool {
exists := false
_, err := os.Stat(filepath.Join(repoPath, "config"))
exists = err == nil
if r.debug {
r.logger.Printf("[DEBUG] [restic] RepoExists: repo=%s, exists=%v", repoPath, exists)
}
return exists
}
// UnlockCommand returns an exec.Cmd that runs restic unlock on the given repo.
func (r *ResticManager) UnlockCommand(ctx context.Context, repoPath string) *exec.Cmd {
return r.command(ctx, repoPath, "unlock")
}
func (r *ResticManager) command(ctx context.Context, repoPath string, args ...string) *exec.Cmd {
if r.debug {
r.logger.Printf("[DEBUG] [restic] command: restic %s (repo=%s)", strings.Join(args, " "), repoPath)
}
cmd := exec.CommandContext(ctx, "restic", args...)
cmd.Env = append(os.Environ(),
"RESTIC_REPOSITORY="+repoPath,
"RESTIC_PASSWORD_FILE="+r.passwordFile,
"RESTIC_CACHE_DIR="+r.cacheDir,
)
return cmd
}
func (r *ResticManager) generatePassword() error {
// Ensure directory exists
dir := filepath.Dir(r.passwordFile)
if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("creating password dir: %w", err)
}
// Generate 32 random bytes, base64url-encode
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return fmt.Errorf("generating random bytes: %w", err)
}
password := base64.URLEncoding.EncodeToString(b)
if err := os.WriteFile(r.passwordFile, []byte(password), 0600); err != nil {
return fmt.Errorf("writing password file: %w", err)
}
r.logger.Printf("[INFO] [backup] Generated new restic repository password at %s", r.passwordFile)
r.logger.Printf("[WARN] [backup] Save this password externally — losing it means losing access to ALL backups")
return nil
}
func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
controller/internal/backup/restore.go
+16 -295
View File
@@ -5,27 +5,24 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
// snapshotIDRe validates restic snapshot IDs: 8-64 lowercase hex characters.
var snapshotIDRe = regexp.MustCompile(`^[0-9a-f]{8,64}$`)
// RestoreApp restores an app from a restic snapshot.
// All apps get config + DB dump restored. Apps with HDD data also get user data restored.
// RestoreApp restores an app's data from its on-disk app-data backup.
//
// Disk-tier (restic snapshot) restore has moved to the host agent. This keep-side
// restore re-imports the Docker-volume tar dumps that the app-data backup produced
// (AppVolumeDumpPath) and relies on the DB dumps already present on the app's drive.
// The stack is stopped before the volume import and restarted after.
//
// snapshotID is retained for API/UI signature compatibility; with restic removed it
// is only used for logging (the source of truth is now the on-disk volume tars).
func (m *Manager) RestoreApp(stackName, snapshotID string) error {
if m.stackProvider == nil {
return fmt.Errorf("stack provider not configured")
}
// Validate snapshot ID format
if !snapshotIDRe.MatchString(snapshotID) {
return fmt.Errorf("invalid snapshot ID: must be 8-64 lowercase hex characters")
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: stack=%s, snapshotID=%s", stackName, snapshotID)
}
@@ -44,87 +41,24 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
m.mu.Unlock()
}()
// Determine what to restore
hddMounts := m.stackProvider.GetStackHDDMounts(stackName)
hasHDD := len(hddMounts) > 0
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: %s has %d HDD mount(s), hasHDD=%v", stackName, len(hddMounts), hasHDD)
}
// Build list of paths to restore from the snapshot
var restorePaths []string
// Always restore the stack's config dir (compose + app.yaml + .felhom.yml)
composePath, ok := m.stackProvider.GetStackComposePath(stackName)
if ok {
stackDir := filepath.Dir(composePath)
restorePaths = append(restorePaths, stackDir)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore config dir: %s", stackDir)
}
}
// Restore DB dump files for this stack (per-drive path)
drivePath := m.GetAppDrivePath(stackName)
dumpDir := AppDBDumpPath(drivePath, stackName)
restorePaths = append(restorePaths, dumpDir)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore DB dump dir: %s", dumpDir)
if drivePath == "" {
return fmt.Errorf("cannot determine drive path for %s", stackName)
}
// Restore HDD data (always included for apps that have it — backup is mandatory)
if hasHDD {
restorePaths = append(restorePaths, hddMounts...)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore HDD data: %v", hddMounts)
}
}
// Restore Docker volume dumps (if present in snapshot)
volDumpDir := AppVolumeDumpPath(drivePath, stackName)
restorePaths = append(restorePaths, volDumpDir)
if len(restorePaths) == 0 {
return fmt.Errorf("no restorable paths found for %s", stackName)
}
// Use the app's primary restic repo
repoPath := PrimaryResticRepoPath(drivePath)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: using repo=%s, %d restore path(s)", repoPath, len(restorePaths))
}
m.logger.Printf("[INFO] [backup] Starting restore for %s (snapshot=%s, repo=%s, paths=%v, hasHDD=%v)",
stackName, snapshotID, repoPath, restorePaths, hasHDD)
m.logger.Printf("[INFO] [backup] Starting app-data restore for %s (drive=%s)", stackName, drivePath)
// Stop the app before restore
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 1/4 — stopping app %s", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 1/3 — stopping app %s", stackName)
}
if err := m.stackProvider.StopStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not stop %s: %v (proceeding anyway)", stackName, err)
}
// Execute restore via restic
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 2/4 — restoring data from snapshot %s", snapshotID)
}
if err := m.restic.RestoreAppData(repoPath, snapshotID, restorePaths); err != nil {
m.logger.Printf("[ERROR] RESTORE failed for %s: %v", stackName, err)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 3/4 — restarting app %s after failure", stackName)
}
if startErr := m.stackProvider.StartStack(stackName); startErr != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after failure: %v", stackName, startErr)
}
return err
}
// Populate Docker volumes from restored tars
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 3/5 — restoring Docker volumes for %s", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 2/3 — restoring Docker volumes for %s", stackName)
}
if err := m.restoreDockerVolumes(stackName, drivePath); err != nil {
m.logger.Printf("[WARN] RESTORE volume restore failed for %s: %v (continuing)", stackName, err)
@@ -132,7 +66,7 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
// Restart the app
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 4/5 — restarting app %s after successful restore", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 3/3 — restarting app %s after restore", stackName)
}
if err := m.stackProvider.StartStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after restore: %v", stackName, err)
@@ -143,219 +77,7 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
m.logger.Printf("[WARN] [backup] Restore completed but app health check failed: %v", err)
}
hasVolumes := len(m.stackProvider.GetDockerVolumes(stackName)) > 0
restoreType := "config+DB"
if hasHDD || hasVolumes {
restoreType = "full (config+DB+userdata)"
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 5/5 — restore completed, type=%s", restoreType)
}
m.logger.Printf("[INFO] RESTORE completed: stack=%s, snapshot=%s, type=%s", stackName, snapshotID, restoreType)
return nil
}
// RestoreAppFromTier2 restores an app from its cross-drive rsync backup mirror.
func (m *Manager) RestoreAppFromTier2(stackName string) error {
if m.stackProvider == nil {
return fmt.Errorf("stack provider not configured")
}
if m.settings == nil {
return fmt.Errorf("settings not available")
}
cdCfg := m.settings.GetCrossDriveConfig(stackName)
if cdCfg == nil || !cdCfg.Enabled {
return fmt.Errorf("cross-drive backup not configured for %s", stackName)
}
rsyncDir := AppSecondaryRsyncPath(cdCfg.DestinationPath, stackName)
if _, err := os.Stat(rsyncDir); os.IsNotExist(err) {
return fmt.Errorf("Tier 2 backup directory not found: %s", rsyncDir)
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: stack=%s, rsyncDir=%s", stackName, rsyncDir)
}
// Prevent concurrent operations
m.mu.Lock()
if m.running {
m.mu.Unlock()
return fmt.Errorf("backup or restore already in progress")
}
m.running = true
m.mu.Unlock()
defer func() {
m.mu.Lock()
m.running = false
m.mu.Unlock()
}()
hddMounts := m.stackProvider.GetStackHDDMounts(stackName)
hasHDD := len(hddMounts) > 0
drivePath := m.GetAppDrivePath(stackName)
m.logger.Printf("[INFO] [backup] Starting Tier 2 restore for %s from %s", stackName, rsyncDir)
// Step 1: Stop the app
if err := m.stackProvider.StopStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not stop %s: %v (proceeding anyway)", stackName, err)
}
// Step 2: Restore config from _config/
configSrc := filepath.Join(rsyncDir, "_config") + "/"
if _, err := os.Stat(filepath.Join(rsyncDir, "_config")); err == nil {
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok {
configDst := filepath.Dir(composePath) + "/"
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync config %s → %s", configSrc, configDst)
}
cmd := exec.Command("rsync", "-a", "--delete", configSrc, configDst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 config restore failed for %s: %v (%s)", stackName, err, strings.TrimSpace(string(out)))
// Try to restart and return error
m.stackProvider.StartStack(stackName)
return fmt.Errorf("config restore failed: %w", err)
}
}
}
// Step 3: Restore HDD data
if hasHDD {
// Check for data directory structure — single mount vs multi-mount
if len(hddMounts) == 1 {
// Single mount: data is directly in rsyncDir (excluding _* dirs)
src := strings.TrimRight(rsyncDir, "/") + "/"
dst := strings.TrimRight(hddMounts[0], "/") + "/"
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync HDD data %s → %s", src, dst)
}
cmd := exec.Command("rsync", "-a", "--delete",
"--exclude", "_*",
src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 HDD data restore failed for %s: %v (%s)", stackName, err, strings.TrimSpace(string(out)))
m.stackProvider.StartStack(stackName)
return fmt.Errorf("HDD data restore failed: %w", err)
}
} else {
// Multiple mounts: each has a subdirectory named by leaf
for _, mount := range hddMounts {
leaf := filepath.Base(mount)
src := filepath.Join(rsyncDir, leaf) + "/"
dst := strings.TrimRight(mount, "/") + "/"
if _, err := os.Stat(filepath.Join(rsyncDir, leaf)); os.IsNotExist(err) {
m.logger.Printf("[WARN] [backup] Tier 2 restore: no backup data for mount %s", mount)
continue
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync HDD mount %s → %s", src, dst)
}
cmd := exec.Command("rsync", "-a", "--delete", src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 HDD restore failed for mount %s: %v (%s)", mount, err, strings.TrimSpace(string(out)))
m.stackProvider.StartStack(stackName)
return fmt.Errorf("HDD restore failed for %s: %w", mount, err)
}
}
}
}
// Step 4: Restore DB dumps from _db/
dbSrc := filepath.Join(rsyncDir, "_db")
if _, err := os.Stat(dbSrc); err == nil {
dbDst := AppDBDumpPath(drivePath, stackName)
if err := os.MkdirAll(dbDst, 0755); err == nil {
entries, _ := os.ReadDir(dbSrc)
for _, e := range entries {
if !e.IsDir() {
src := filepath.Join(dbSrc, e.Name())
dst := filepath.Join(dbDst, e.Name())
if err := copyFile(src, dst); err != nil {
m.logger.Printf("[WARN] [backup] Failed to copy DB dump %s: %v", e.Name(), err)
}
}
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: restored DB dumps from %s", dbSrc)
}
}
}
// Step 5: Restore Docker volumes from _volumes/
volSrc := filepath.Join(rsyncDir, "_volumes")
if _, err := os.Stat(volSrc); err == nil {
if err := m.restoreDockerVolumesFromDir(stackName, volSrc); err != nil {
m.logger.Printf("[WARN] [backup] Tier 2 volume restore failed for %s: %v (continuing)", stackName, err)
}
}
// Step 6: Restart the app
if err := m.stackProvider.StartStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after Tier 2 restore: %v", stackName, err)
}
// Verify app started successfully
if err := m.waitForHealthy(stackName, 90*time.Second); err != nil {
m.logger.Printf("[WARN] [backup] Tier 2 restore completed but app health check failed: %v", err)
}
hasVolumes := len(m.stackProvider.GetDockerVolumes(stackName)) > 0
restoreType := "config+DB"
if hasHDD || hasVolumes {
restoreType = "full (config+DB+userdata)"
}
m.logger.Printf("[INFO] RESTORE (Tier 2) completed: stack=%s, type=%s", stackName, restoreType)
return nil
}
// restoreDockerVolumesFromDir populates Docker volumes from tar files in an arbitrary directory.
// Used by Tier 2 restore where volume tars are in the rsync mirror's _volumes/ dir.
func (m *Manager) restoreDockerVolumesFromDir(stackName, dumpDir string) error {
entries, err := os.ReadDir(dumpDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("reading volume dump dir: %w", err)
}
var restored int
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tar") {
continue
}
volName := strings.TrimSuffix(entry.Name(), ".tar")
m.logger.Printf("[INFO] [backup] Restoring Docker volume %s for %s (Tier 2)", volName, stackName)
exec.Command("docker", "volume", "rm", "-f", volName).Run()
if out, err := exec.Command("docker", "volume", "create", volName).CombinedOutput(); err != nil {
m.logger.Printf("[WARN] [backup] Failed to create volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err)
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
"-v", volName+":/vol",
"-v", dumpDir+":/in:ro",
"alpine", "tar", "xf", "/in/"+entry.Name(), "-C", "/vol")
out, err := cmd.CombinedOutput()
cancel()
if err != nil {
m.logger.Printf("[WARN] [backup] Failed to populate volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err)
continue
}
restored++
}
if restored > 0 {
m.logger.Printf("[INFO] [backup] Restored %d Docker volume(s) for %s (Tier 2)", restored, stackName)
}
m.logger.Printf("[INFO] RESTORE completed: stack=%s", stackName)
return nil
}
@@ -416,7 +138,6 @@ func (m *Manager) restoreDockerVolumes(stackName, drivePath string) error {
// waitForHealthy waits for a stack to reach running state after restore.
// Forces a docker ps refresh on each poll to avoid stale state.
// Acceptable overhead for a rare operation (restore).
func (m *Manager) waitForHealthy(stackName string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
interval := 5 * time.Second
controller/internal/backup/restore_app_linux.go
-246
View File
@@ -1,246 +0,0 @@
//go:build linux
package backup
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// RestoreAppFromBackup restores a single app from its cross-drive backup.
// Steps: restore config → verify/restore data → copy DB dumps → docker compose up.
func RestoreAppFromBackup(ctx context.Context, app *RestorableApp, stacksDir string, logger *log.Logger) error {
stackDir := filepath.Join(stacksDir, app.Name)
start := time.Now()
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: app=%s, stackDir=%s, hasConfig=%v, hasData=%v, hasDBDump=%v, hasRsyncData=%v",
app.Name, stackDir, app.HasConfig, app.HasData, app.HasDBDump, app.HasRsyncData)
// Step 1: Restore stack config from _config/ backup
if app.HasConfig {
logger.Printf("[INFO] Restoring config for %s from %s", app.Name, app.ConfigPath)
stepStart := time.Now()
if err := restoreConfigDir(ctx, app.ConfigPath, stackDir); err != nil {
return fmt.Errorf("restoring config: %w", err)
}
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: config restore for %s completed in %s", app.Name, time.Since(stepStart))
} else {
// No config backup — check if stack dir already exists (from catalog sync)
if !dirExists(stackDir) {
return fmt.Errorf("no config backup and no stack directory for %s", app.Name)
}
logger.Printf("[INFO] No config backup for %s — using existing stack dir", app.Name)
}
// Step 2: Verify app data on HDD (common case: HDD survived, data is intact)
if app.NeedsHDD && !app.HasData && app.HasRsyncData {
// App data is missing but rsync backup exists — restore it
logger.Printf("[INFO] Restoring user data for %s from rsync backup", app.Name)
stepStart := time.Now()
if err := restoreUserData(ctx, app, logger); err != nil {
logger.Printf("[WARN] User data restore failed for %s: %v", app.Name, err)
// Non-fatal: app might still start without all data
} else {
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: user data restore for %s completed in %s", app.Name, time.Since(stepStart))
}
} else if app.HasData {
logger.Printf("[INFO] App data for %s found at %s — no restore needed", app.Name, app.DataPath)
} else {
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: %s — no user data to restore (needsHDD=%v, hasData=%v, hasRsyncData=%v)",
app.Name, app.NeedsHDD, app.HasData, app.HasRsyncData)
}
// Step 3: Copy DB dumps to primary backup location
if app.HasDBDump {
logger.Printf("[INFO] Restoring DB dumps for %s", app.Name)
stepStart := time.Now()
if err := restoreDBDumps(app, logger); err != nil {
logger.Printf("[WARN] DB dump restore failed for %s: %v", app.Name, err)
// Non-fatal
} else {
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: DB dump restore for %s completed in %s", app.Name, time.Since(stepStart))
}
}
// Step 4: Docker compose pull + up
composePath := filepath.Join(stackDir, "docker-compose.yml")
if !fileExistsCheck(composePath) {
composePath = filepath.Join(stackDir, "compose.yml")
if !fileExistsCheck(composePath) {
return fmt.Errorf("no compose file found in %s", stackDir)
}
}
composeDir := filepath.Dir(composePath)
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: %s using compose file %s", app.Name, composePath)
logger.Printf("[INFO] Pulling images for %s", app.Name)
pullStart := time.Now()
pullCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "pull")
pullCmd.Dir = composeDir
if out, err := pullCmd.CombinedOutput(); err != nil {
logger.Printf("[WARN] docker compose pull failed for %s: %v (%s)", app.Name, err, strings.TrimSpace(string(out)))
// Non-fatal: might work with cached images
} else {
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: docker compose pull for %s completed in %s", app.Name, time.Since(pullStart))
}
logger.Printf("[INFO] Starting %s", app.Name)
upStart := time.Now()
upCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "up", "-d")
upCmd.Dir = composeDir
if out, err := upCmd.CombinedOutput(); err != nil {
return fmt.Errorf("docker compose up: %v (%s)", err, strings.TrimSpace(string(out)))
}
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: %s fully restored and started in %s", app.Name, time.Since(start))
logger.Printf("[DEBUG] [backup] RestoreAppFromBackup: docker compose up for %s completed in %s", app.Name, time.Since(upStart))
return nil
}
// restoreConfigDir rsyncs the backed-up _config/ directory to the stack directory.
func restoreConfigDir(ctx context.Context, configBackupDir, stackDir string) error {
if err := os.MkdirAll(stackDir, 0755); err != nil {
return fmt.Errorf("creating stack dir: %w", err)
}
src := strings.TrimRight(configBackupDir, "/") + "/"
dst := strings.TrimRight(stackDir, "/") + "/"
cmd := exec.CommandContext(ctx, "rsync", "-a", src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("rsync config: %v (%s)", err, strings.TrimSpace(string(out)))
}
return nil
}
// restoreUserData rsyncs user data from cross-drive backup back to the app's HDD path.
func restoreUserData(ctx context.Context, app *RestorableApp, logger *log.Logger) error {
if app.RsyncDataPath == "" || app.HDDPath == "" {
return fmt.Errorf("no rsync data path or HDD path")
}
logger.Printf("[DEBUG] [backup] restoreUserData: app=%s, rsyncPath=%s, hddPath=%s", app.Name, app.RsyncDataPath, app.HDDPath)
// The rsync backup contains the app's data directories.
// Walk the backup dir and rsync each subdirectory (excluding _config/_db)
// back to the app's HDD data directory.
entries, err := os.ReadDir(app.RsyncDataPath)
if err != nil {
return err
}
dataDir := AppDataDir(app.HDDPath, app.Name)
logger.Printf("[DEBUG] [backup] restoreUserData: %s — target dataDir=%s, %d entries in backup", app.Name, dataDir, len(entries))
if err := os.MkdirAll(dataDir, 0755); err != nil {
return fmt.Errorf("creating data dir: %w", err)
}
restored := 0
for _, e := range entries {
name := e.Name()
if name == "_config" || name == "_db" || strings.HasPrefix(name, ".") {
continue
}
src := filepath.Join(app.RsyncDataPath, name)
dst := filepath.Join(dataDir, name)
if e.IsDir() {
src = strings.TrimRight(src, "/") + "/"
if err := os.MkdirAll(dst, 0755); err != nil {
logger.Printf("[ERROR] [backup] Cannot create %s: %v", dst, err)
continue
}
dst = strings.TrimRight(dst, "/") + "/"
logger.Printf("[DEBUG] [backup] restoreUserData: %s — rsync dir %s → %s", app.Name, src, dst)
cmd := exec.CommandContext(ctx, "rsync", "-a", src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
logger.Printf("[ERROR] [backup] rsync data %s: %v (%s)", name, err, strings.TrimSpace(string(out)))
} else {
restored++
}
} else {
// Single file — copy directly
data, err := os.ReadFile(src)
if err != nil {
logger.Printf("[ERROR] [backup] Cannot read %s: %v", src, err)
continue
}
logger.Printf("[DEBUG] [backup] restoreUserData: %s — copying file %s (%d bytes)", app.Name, name, len(data))
if err := os.WriteFile(dst, data, 0644); err != nil {
logger.Printf("[ERROR] [backup] Cannot write %s: %v", dst, err)
} else {
restored++
}
}
}
logger.Printf("[DEBUG] [backup] restoreUserData: %s — restored %d items", app.Name, restored)
return nil
}
// restoreDBDumps copies DB dump files from cross-drive backup to the primary dump dir.
func restoreDBDumps(app *RestorableApp, logger *log.Logger) error {
if app.DBDumpPath == "" {
return nil
}
// Use HDDPath for apps with HDD data, fall back to DrivePath (system data path)
// for SSD-only apps whose DB dumps live under the system drive.
drivePath := app.HDDPath
if drivePath == "" {
drivePath = app.DrivePath
}
if drivePath == "" {
logger.Printf("[WARN] Cannot restore DB dumps for %s: no drive path", app.Name)
return nil
}
destDir := AppDBDumpPath(drivePath, app.Name)
logger.Printf("[DEBUG] [backup] restoreDBDumps: app=%s, src=%s, destDir=%s", app.Name, app.DBDumpPath, destDir)
if err := os.MkdirAll(destDir, 0755); err != nil {
return fmt.Errorf("creating dump dir: %w", err)
}
entries, err := os.ReadDir(app.DBDumpPath)
if err != nil {
return err
}
copied := 0
for _, e := range entries {
if e.IsDir() {
continue
}
src := filepath.Join(app.DBDumpPath, e.Name())
dst := filepath.Join(destDir, e.Name())
data, err := os.ReadFile(src)
if err != nil {
logger.Printf("[ERROR] [backup] Cannot read dump %s: %v", e.Name(), err)
continue
}
logger.Printf("[DEBUG] [backup] restoreDBDumps: %s — copying %s (%d bytes)", app.Name, e.Name(), len(data))
if err := os.WriteFile(dst, data, 0644); err != nil {
logger.Printf("[ERROR] [backup] Cannot write dump %s: %v", e.Name(), err)
} else {
copied++
}
}
logger.Printf("[DEBUG] [backup] restoreDBDumps: %s — copied %d dump files", app.Name, copied)
return nil
}
// fileExistsCheck returns true if path exists and is a file.
func fileExistsCheck(path string) bool {
info, err := os.Stat(path)
return err == nil && !info.IsDir()
}
controller/internal/backup/restore_app_other.go
-13
View File
@@ -1,13 +0,0 @@
//go:build !linux
package backup
import (
"context"
"log"
)
// RestoreAppFromBackup is a no-op on non-Linux platforms.
func RestoreAppFromBackup(ctx context.Context, app *RestorableApp, stacksDir string, logger *log.Logger) error {
return nil
}
controller/internal/backup/restore_drives_linux.go
-325
View File
@@ -1,325 +0,0 @@
//go:build linux
package backup
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
// MountDrivesFromLayout scans block devices for disks matching the stored
// disk layout and mounts them using the felhom two-layer mount pattern
// (raw mount → bind mount).
//
// The controller container runs privileged with:
// - /host-dev mounted from host /dev
// - /host-fstab mounted from host /etc/fstab
// - /mnt with rshared propagation
//
// Returns the list of successfully mounted final mount paths.
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
if len(layout.Mounts) == 0 {
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: no mounts in layout, nothing to do")
return nil, nil
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %d mount entries from disk layout", len(layout.Mounts))
// Get current block devices with UUIDs
uuidToDevice, err := scanBlockDeviceUUIDs(ctx)
if err != nil {
return nil, fmt.Errorf("scanning block devices: %w", err)
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: discovered %d block devices with UUIDs", len(uuidToDevice))
for uuid, dev := range uuidToDevice {
uuidShort := uuid
if len(uuidShort) > 12 {
uuidShort = uuidShort[:12]
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: device %s → UUID=%s...", dev, uuidShort)
}
var mounted []string
for _, dm := range layout.Mounts {
if dm.UUID == "" {
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: skipping mount entry with empty UUID (label=%s)", dm.Label)
continue
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: processing %s (UUID=%s, mountPoint=%s, rawMount=%s, fsType=%s)",
dm.Label, dm.UUID, dm.MountPoint, dm.RawMount, dm.FSType)
// Find matching device by UUID
device := uuidToDevice[dm.UUID]
if device == "" {
logger.Printf("[WARN] Disk UUID %s (%s) not found — drive may be missing or disconnected",
dm.UUID, dm.Label)
continue
}
// Check if already mounted
finalMount := dm.MountPoint
if isMountedPath(finalMount) {
logger.Printf("[INFO] %s already mounted at %s", dm.Label, finalMount)
mounted = append(mounted, finalMount)
continue
}
if dm.RawMount != "" && isMountedPath(dm.RawMount) {
logger.Printf("[INFO] %s raw mount already at %s", dm.Label, dm.RawMount)
mounted = append(mounted, finalMount)
continue
}
uuidShort := dm.UUID
if len(uuidShort) > 12 {
uuidShort = uuidShort[:12]
}
logger.Printf("[INFO] Found disk %s (UUID=%s, label=%s) — mounting to %s",
device, uuidShort, dm.Label, finalMount)
// Mount using the appropriate pattern
if dm.RawMount != "" && dm.BindSubdir != "" {
// Two-layer HDD mount: raw → bind
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — two-layer mount (raw=%s, bindSubdir=%s)",
dm.Label, dm.RawMount, dm.BindSubdir)
if err := mountRawAndBind(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
} else {
// Simple direct mount (e.g., sys_drive)
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: %s — direct mount to %s", dm.Label, dm.MountPoint)
if err := mountDirect(ctx, device, dm, logger); err != nil {
logger.Printf("[ERROR] Failed to mount %s: %v", dm.Label, err)
continue
}
}
// Update host fstab so mount persists across reboots
if err := addDRFstabEntries(dm, logger); err != nil {
logger.Printf("[WARN] Failed to update fstab for %s: %v — mount works but won't persist", dm.Label, err)
}
mounted = append(mounted, finalMount)
logger.Printf("[INFO] Successfully mounted %s at %s", dm.Label, finalMount)
}
logger.Printf("[DEBUG] [backup] MountDrivesFromLayout: done — %d/%d drives mounted", len(mounted), len(layout.Mounts))
return mounted, nil
}
// scanBlockDeviceUUIDs runs lsblk + blkid to build a UUID -> device path map.
func scanBlockDeviceUUIDs(ctx context.Context) (map[string]string, error) {
// First try lsblk with UUID output
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,UUID,FSTYPE,MOUNTPOINT").Output()
if err != nil {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
var parsed struct {
BlockDevices []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
Children []struct {
Name string `json:"name"`
UUID *string `json:"uuid"`
FSType *string `json:"fstype"`
Mount *string `json:"mountpoint"`
} `json:"children"`
} `json:"blockdevices"`
}
if err := json.Unmarshal(out, &parsed); err != nil {
return nil, fmt.Errorf("lsblk parse failed: %w", err)
}
devices := make(map[string]string) // UUID → /dev/path
for _, dev := range parsed.BlockDevices {
if dev.UUID != nil && *dev.UUID != "" {
devices[*dev.UUID] = "/dev/" + dev.Name
}
for _, child := range dev.Children {
if child.UUID != nil && *child.UUID != "" {
devices[*child.UUID] = "/dev/" + child.Name
}
}
}
// If lsblk didn't return UUIDs (common inside containers), enrich via blkid
if len(devices) == 0 {
// Try blkid on /host-dev devices
blkOut, err := exec.CommandContext(ctx, "blkid").Output()
if err == nil {
for _, line := range strings.Split(string(blkOut), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Parse: /dev/sdb1: UUID="277a2179-..." TYPE="ext4" ...
colonIdx := strings.Index(line, ":")
if colonIdx < 0 {
continue
}
devPath := line[:colonIdx]
if uuidIdx := strings.Index(line, `UUID="`); uuidIdx >= 0 {
rest := line[uuidIdx+6:]
if endIdx := strings.Index(rest, `"`); endIdx >= 0 {
uuid := rest[:endIdx]
devices[uuid] = devPath
}
}
}
}
}
return devices, nil
}
// mountDirect creates a simple direct mount.
func mountDirect(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating mount point: %w", err)
}
// Use host device path if available
devPath := hostDevPath(device)
logger.Printf("[DEBUG] [backup] mountDirect: mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.MountPoint)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount %s: %s: %w", devPath, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountDirect: %s mounted successfully at %s", devPath, dm.MountPoint)
return nil
}
// mountRawAndBind implements the two-layer felhom mount pattern.
func mountRawAndBind(ctx context.Context, device string, dm DiskMount, logger *log.Logger) error {
// Layer 1: raw mount
if err := os.MkdirAll(dm.RawMount, 0755); err != nil {
return fmt.Errorf("creating raw mount point: %w", err)
}
devPath := hostDevPath(device)
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 — mount -t %s -o noatime %s %s", dm.FSType, devPath, dm.RawMount)
cmd := exec.CommandContext(ctx, "mount", "-t", dm.FSType, "-o", "noatime", devPath, dm.RawMount)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("raw mount %s -> %s: %s: %w", devPath, dm.RawMount, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 1 OK — %s mounted at %s", devPath, dm.RawMount)
// Layer 2: bind mount (subdir -> final mount point)
bindSrc := filepath.Join(dm.RawMount, dm.BindSubdir)
if err := os.MkdirAll(bindSrc, 0755); err != nil {
return fmt.Errorf("creating bind source dir: %w", err)
}
if err := os.MkdirAll(dm.MountPoint, 0755); err != nil {
return fmt.Errorf("creating final mount point: %w", err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 — mount --bind %s %s", bindSrc, dm.MountPoint)
cmd = exec.CommandContext(ctx, "mount", "--bind", bindSrc, dm.MountPoint)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("bind mount %s -> %s: %s: %w", bindSrc, dm.MountPoint, strings.TrimSpace(string(out)), err)
}
logger.Printf("[DEBUG] [backup] mountRawAndBind: layer 2 OK — %s bound to %s", bindSrc, dm.MountPoint)
return nil
}
// addDRFstabEntries adds fstab entries so mounts persist across host reboots.
func addDRFstabEntries(dm DiskMount, logger *log.Logger) error {
const fstabPath = "/host-fstab"
logger.Printf("[INFO] [backup] Adding fstab entries for disaster recovery (%s, UUID=%s)", dm.Label, dm.UUID)
logger.Printf("[DEBUG] [backup] addDRFstabEntries: checking fstab for %s (UUID=%s)", dm.Label, dm.UUID)
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("reading fstab: %w", err)
}
content := string(data)
// Skip if UUID already in fstab (idempotent)
if strings.Contains(content, dm.UUID) {
logger.Printf("[DEBUG] [backup] addDRFstabEntries: UUID %s already in fstab — skipping", dm.UUID)
return nil
}
var additions strings.Builder
additions.WriteString("\n# Restored by felhom-controller DR\n")
entryCount := 0
if dm.RawMount != "" {
// Raw mount entry
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.RawMount, dm.FSType, dm.FstabOptions))
entryCount++
}
if dm.BindSubdir != "" && dm.RawMount != "" {
// Bind mount entry
additions.WriteString(fmt.Sprintf("%s/%s\t%s\tnone\tbind,nofail\t0 0\n",
dm.RawMount, dm.BindSubdir, dm.MountPoint))
entryCount++
} else if dm.RawMount == "" {
// Direct mount entry (no bind)
additions.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0 2\n",
dm.UUID, dm.MountPoint, dm.FSType, dm.FstabOptions))
entryCount++
}
newContent := content + additions.String()
// Write atomically (try rename, fallback to direct write for bind-mounted fstab)
tmpPath := fstabPath + ".tmp"
if err := os.WriteFile(tmpPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab tmp: %w", err)
}
if err := os.Rename(tmpPath, fstabPath); err != nil {
os.Remove(tmpPath)
// Fallback: direct write (bind-mounted files can't be renamed)
if err := os.WriteFile(fstabPath, []byte(newContent), 0644); err != nil {
return fmt.Errorf("writing fstab: %w", err)
}
}
logger.Printf("[INFO] [backup] Added %d fstab entries for %s", entryCount, dm.Label)
return nil
}
// isMountedPath checks if a path is currently a mount point via /proc/mounts.
func isMountedPath(path string) bool {
if path == "" {
return false
}
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false
}
cleanPath := filepath.Clean(path)
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && filepath.Clean(fields[1]) == cleanPath {
return true
}
}
return false
}
// hostDevPath converts /dev/xxx to /host-dev/xxx for container access.
func hostDevPath(devPath string) string {
if strings.HasPrefix(devPath, "/dev/") {
return "/host-dev/" + strings.TrimPrefix(devPath, "/dev/")
}
return devPath
}
controller/internal/backup/restore_drives_other.go
-13
View File
@@ -1,13 +0,0 @@
//go:build !linux
package backup
import (
"context"
"log"
)
// MountDrivesFromLayout is a no-op on non-Linux platforms.
func MountDrivesFromLayout(ctx context.Context, layout DiskLayout, logger *log.Logger) ([]string, error) {
return nil, nil
}
controller/internal/backup/restore_scan.go
-310
View File
@@ -1,310 +0,0 @@
package backup
import (
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
// RestorableApp describes an app that can be restored during DR.
type RestorableApp struct {
Name string `json:"name"`
DisplayName string `json:"display_name"`
NeedsHDD bool `json:"needs_hdd"`
HDDPath string `json:"hdd_path,omitempty"`
// What was found on disk
HasConfig bool `json:"has_config"` // _config/ dir with compose files
ConfigPath string `json:"config_path"` // full path to _config/ backup
HasData bool `json:"has_data"` // app data dir exists on HDD
DataPath string `json:"data_path"` // e.g., /mnt/hdd_1/appdata/immich
HasDBDump bool `json:"has_db_dump"` // _db/ dir with dump files
DBDumpPath string `json:"db_dump_path"` // full path to _db/ backup
HasRsyncData bool `json:"has_rsync_data"` // rsync user data (excl _config/_db)
RsyncDataPath string `json:"rsync_data_path"` // full path to rsync backup
DrivePath string `json:"drive_path"` // which drive has the backup
DriveLabel string `json:"drive_label"` // label for display
// Restore progress (updated during restore)
Status string `json:"status"` // "pending", "restoring", "done", "failed", "skipped"
Error string `json:"error,omitempty"`
StartedAt string `json:"started_at,omitempty"`
CompletedAt string `json:"completed_at,omitempty"`
}
// RestorePlan holds the complete DR restore plan.
type RestorePlan struct {
mu sync.RWMutex
CustomerID string `json:"customer_id"`
Domain string `json:"domain"`
Timestamp string `json:"timestamp"` // when the infra backup was taken
Apps []RestorableApp `json:"apps"`
// Drive summary
Drives []DriveInfo `json:"drives"`
// Overall status
Status string `json:"status"` // "pending", "restoring", "done"
}
// DriveInfo summarizes a mounted drive for display.
type DriveInfo struct {
Path string `json:"path"`
Label string `json:"label"`
Available bool `json:"available"` // mount is accessible
HasBackup bool `json:"has_backup"` // has backups/secondary/ dir
}
// GetApps returns a snapshot of the apps list.
func (rp *RestorePlan) GetApps() []RestorableApp {
rp.mu.RLock()
defer rp.mu.RUnlock()
apps := make([]RestorableApp, len(rp.Apps))
copy(apps, rp.Apps)
return apps
}
// Snapshot returns a thread-safe snapshot of the plan for JSON serialization.
func (rp *RestorePlan) Snapshot() map[string]interface{} {
rp.mu.RLock()
defer rp.mu.RUnlock()
apps := make([]RestorableApp, len(rp.Apps))
copy(apps, rp.Apps)
drives := make([]DriveInfo, len(rp.Drives))
copy(drives, rp.Drives)
return map[string]interface{}{
"ok": true,
"status": rp.Status,
"apps": apps,
"drives": drives,
}
}
// TryStartRestore atomically sets status to "restoring" if not already restoring.
// Returns false if a restore is already in progress (prevents double-restore race).
func (rp *RestorePlan) TryStartRestore() bool {
rp.mu.Lock()
defer rp.mu.Unlock()
if rp.Status == "restoring" {
return false
}
rp.Status = "restoring"
return true
}
// SetStatus sets the overall plan status under lock.
func (rp *RestorePlan) SetStatus(status string) {
rp.mu.Lock()
defer rp.mu.Unlock()
rp.Status = status
}
// GetStatus returns the current plan status under lock.
func (rp *RestorePlan) GetStatus() string {
rp.mu.RLock()
defer rp.mu.RUnlock()
return rp.Status
}
// UpdateApp updates a single app's status in the plan.
func (rp *RestorePlan) UpdateApp(name, status, errMsg string) {
rp.mu.Lock()
defer rp.mu.Unlock()
for i := range rp.Apps {
if rp.Apps[i].Name == name {
rp.Apps[i].Status = status
rp.Apps[i].Error = errMsg
if status == "restoring" {
rp.Apps[i].StartedAt = time.Now().UTC().Format(time.RFC3339)
}
if status == "done" || status == "failed" {
rp.Apps[i].CompletedAt = time.Now().UTC().Format(time.RFC3339)
}
return
}
}
}
// AllDone returns true if all apps are done/failed/skipped.
// Returns false for empty plans (no apps to restore).
func (rp *RestorePlan) AllDone() bool {
rp.mu.RLock()
defer rp.mu.RUnlock()
if len(rp.Apps) == 0 {
return false
}
for _, app := range rp.Apps {
if app.Status != "done" && app.Status != "failed" && app.Status != "skipped" {
return false
}
}
return true
}
// InfraStackInfo is a minimal stack descriptor from the Hub infra backup.
// Used to pass deployed_stacks info into the scan without importing report.
type InfraStackInfo struct {
Name string
DisplayName string
HDDPath string
NeedsHDD bool
}
// ScanDrivesForBackups scans mounted drives for cross-drive backup data
// and correlates with the deployed stacks manifest from the Hub.
func ScanDrivesForBackups(mountedPaths []string, stacks []InfraStackInfo, logger *log.Logger) *RestorePlan {
plan := &RestorePlan{
Status: "pending",
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning %d mount paths, %d stacks from manifest",
len(mountedPaths), len(stacks))
// Build drive info and find backup directories
type driveBackup struct {
drivePath string
label string
secPath string // backups/secondary/ path
}
var backupDrives []driveBackup
for _, mp := range mountedPaths {
label := filepath.Base(mp)
avail := dirExists(mp)
di := DriveInfo{
Path: mp,
Label: label,
Available: avail,
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: checking drive %s (label=%s, available=%v)", mp, label, avail)
secPath := SecondaryBackupPath(mp)
if dirExists(secPath) {
di.HasBackup = true
backupDrives = append(backupDrives, driveBackup{
drivePath: mp,
label: label,
secPath: secPath,
})
logger.Printf("[INFO] Found backup data on %s (%s)", mp, secPath)
}
plan.Drives = append(plan.Drives, di)
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: found %d drives with backup data", len(backupDrives))
// For each stack from the manifest, look for backup data on drives
for _, stack := range stacks {
app := RestorableApp{
Name: stack.Name,
DisplayName: stack.DisplayName,
NeedsHDD: stack.NeedsHDD,
HDDPath: stack.HDDPath,
Status: "pending",
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: scanning for app %s (needsHDD=%v, hddPath=%s)",
stack.Name, stack.NeedsHDD, stack.HDDPath)
// Check if app data exists directly on HDD (common case: HDD survived)
if stack.HDDPath != "" {
dataDir := AppDataDir(stack.HDDPath, stack.Name)
if dirExists(dataDir) {
app.HasData = true
app.DataPath = dataDir
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — live data found at %s", stack.Name, dataDir)
}
}
// Scan each drive for cross-drive backup of this app
for _, db := range backupDrives {
rsyncBase := AppSecondaryRsyncPath(db.drivePath, stack.Name)
if !dirExists(rsyncBase) {
continue
}
// Found a backup for this app
app.DrivePath = db.drivePath
app.DriveLabel = db.label
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — backup found on drive %s at %s",
stack.Name, db.label, rsyncBase)
// Check for _config/ (stack compose directory backup)
configDir := filepath.Join(rsyncBase, "_config")
if dirExists(configDir) {
app.HasConfig = true
app.ConfigPath = configDir
}
// Check for _db/ (database dump backup)
dbDir := filepath.Join(rsyncBase, "_db")
if dirExists(dbDir) && !dirIsEmpty(dbDir) {
app.HasDBDump = true
app.DBDumpPath = dbDir
}
// Check for user data in rsync (anything besides _config and _db)
if hasUserData(rsyncBase) {
app.HasRsyncData = true
app.RsyncDataPath = rsyncBase
}
logger.Printf("[DEBUG] [backup] ScanDrivesForBackups: %s — config=%v, dbDump=%v, rsyncData=%v",
stack.Name, app.HasConfig, app.HasDBDump, app.HasRsyncData)
break // use first drive with backup for this app
}
plan.Apps = append(plan.Apps, app)
}
if len(plan.Apps) == 0 {
plan.Apps = []RestorableApp{}
}
logger.Printf("[INFO] Restore plan: %d apps, %d drives (%d with backups)",
len(plan.Apps), len(plan.Drives), len(backupDrives))
return plan
}
// dirExists checks if a directory exists and is accessible.
func dirExists(path string) bool {
info, err := os.Stat(path)
return err == nil && info.IsDir()
}
// dirIsEmpty returns true if a directory has no entries.
// Returns false on read errors (assume non-empty — safer for backup detection).
func dirIsEmpty(path string) bool {
entries, err := os.ReadDir(path)
if err != nil {
return false
}
return len(entries) == 0
}
// hasUserData checks if the rsync backup dir has user data (not just _config/_db).
func hasUserData(rsyncBase string) bool {
entries, err := os.ReadDir(rsyncBase)
if err != nil {
return false
}
for _, e := range entries {
name := e.Name()
if name != "_config" && name != "_db" && !strings.HasPrefix(name, ".") {
return true
}
}
return false
}
controller/internal/monitor/pinger.go
-120
View File
@@ -1,120 +0,0 @@
package monitor
import (
"fmt"
"io"
"log"
"net/http"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)
// Pinger sends health check pings to a Healthchecks.io-compatible server.
type Pinger struct {
baseURL string
httpClient *http.Client
logger *log.Logger
enabled bool
debug bool
}
// NewPinger creates a new Pinger from monitoring config.
func NewPinger(cfg *config.MonitoringConfig, logger *log.Logger) *Pinger {
return &Pinger{
baseURL: strings.TrimRight(cfg.HealthchecksBase, "/"),
httpClient: &http.Client{
Timeout: 10 * time.Second,
},
logger: logger,
enabled: cfg.Enabled,
}
}
// SetDebug enables or disables debug logging for the pinger.
func (p *Pinger) SetDebug(debug bool) {
p.debug = debug
}
// Ping sends a success signal with optional diagnostic body.
func (p *Pinger) Ping(uuid string, body string) error {
if p.debug {
p.logger.Printf("[DEBUG] [pinger] Ping uuid=%s body_len=%d", uuid, len(body))
}
return p.send(uuid, "", body)
}
// Fail sends a failure signal with diagnostic body.
func (p *Pinger) Fail(uuid string, body string) error {
if p.debug {
p.logger.Printf("[DEBUG] [pinger] Fail uuid=%s body=%q", uuid, body)
}
return p.send(uuid, "/fail", body)
}
// Start sends a "job started" signal (for duration tracking).
func (p *Pinger) Start(uuid string) error {
if p.debug {
p.logger.Printf("[DEBUG] [pinger] Start uuid=%s", uuid)
}
return p.send(uuid, "/start", "")
}
func (p *Pinger) send(uuid, suffix, body string) error {
if !p.enabled {
return nil
}
if uuid == "" || strings.HasPrefix(uuid, "CHANGEME") {
return nil
}
url := fmt.Sprintf("%s/ping/%s%s", p.baseURL, uuid, suffix)
if p.debug {
p.logger.Printf("[DEBUG] [pinger] send url=%s", url)
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
if p.debug {
p.logger.Printf("[DEBUG] [pinger] retry attempt=%d uuid=%s", attempt+1, uuid)
}
time.Sleep(2 * time.Second)
}
var bodyReader io.Reader
if body != "" {
bodyReader = strings.NewReader(body)
}
req, err := http.NewRequest(http.MethodPost, url, bodyReader)
if err != nil {
lastErr = err
continue
}
resp, err := p.httpClient.Do(req)
if err != nil {
lastErr = err
continue
}
resp.Body.Close()
if p.debug {
p.logger.Printf("[DEBUG] [pinger] response status=%d uuid=%s", resp.StatusCode, uuid)
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
if p.debug {
p.logger.Printf("[DEBUG] [pinger] success uuid=%s", uuid)
}
return nil
}
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
}
p.logger.Printf("[WARN] [monitor] Health ping failed after 3 attempts (%s): %v", uuid, lastErr)
return nil // Never let ping failures affect the caller
}
controller/internal/monitor/watchdog.go
-902
View File
@@ -1,902 +0,0 @@
package monitor
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
)
const (
// probeThreshold is the number of consecutive probe failures before declaring disconnected.
probeThreshold = 3
// defaultProbeInterval is the normal probe interval for connected drives.
defaultProbeInterval = 5 * time.Second
// disconnectedProbeInterval is the slower probe interval for disconnected drives
// (checking for UUID reappearance, not I/O probing).
disconnectedProbeInterval = 30 * time.Second
// hostFstabPath is where the host's fstab is mounted inside the container.
hostFstabPath = "/host-fstab"
// hostDevUUIDPath is where the host's /dev/disk/by-uuid is accessible.
hostDevUUIDPath = "/host-dev/disk/by-uuid"
// primaryResticSubpath is the relative path to the primary restic repo under a drive.
primaryResticSubpath = "backups/primary/restic"
)
// WatchdogStackInfo holds minimal stack info for the watchdog.
type WatchdogStackInfo struct {
Name string
}
// WatchdogStackProvider provides stack operations needed by the watchdog.
// Defined here to avoid circular imports with the backup package.
type WatchdogStackProvider interface {
ListDeployedStacks() []WatchdogStackInfo
GetStackHDDPath(name string) string
StopStack(name string) error
StartStack(name string) error
}
// pathProbeState tracks in-memory probe state for a single storage path.
type pathProbeState struct {
mu sync.Mutex
consecutiveFailures int
lastStatus string // "connected", "disconnected"
lastProbeTime time.Time
probeInterval time.Duration
// Debug counters for summary logging
probeCount int
probeOKCount int
lastSummaryTime time.Time
totalLatency time.Duration
}
// StorageWatchdog monitors registered storage paths and reacts to disconnection/reconnection.
type StorageWatchdog struct {
settings *settings.Settings
stackProvider WatchdogStackProvider
notifier *notify.Notifier
cfg *config.Config
logger *log.Logger
// Callbacks to break import cycles — set via SetXxx methods after construction
alertRefresh func()
pushHubReport func()
unlockRepo func(ctx context.Context, repoPath string) error
mu sync.Mutex
pathState map[string]*pathProbeState
// Debug simulation state
simulatedMu sync.RWMutex
simulatedPaths map[string]bool
}
// NewStorageWatchdog creates a new storage watchdog.
func NewStorageWatchdog(
sett *settings.Settings,
stackProvider WatchdogStackProvider,
notifier *notify.Notifier,
cfg *config.Config,
logger *log.Logger,
) *StorageWatchdog {
return &StorageWatchdog{
settings: sett,
stackProvider: stackProvider,
notifier: notifier,
cfg: cfg,
logger: logger,
pathState: make(map[string]*pathProbeState),
simulatedPaths: make(map[string]bool),
}
}
// isDebug returns true if the logging level is set to "debug".
func (w *StorageWatchdog) isDebug() bool { return w.cfg.Logging.Level == "debug" }
// SetAlertRefresh sets the callback to trigger alert refresh.
func (w *StorageWatchdog) SetAlertRefresh(fn func()) {
w.alertRefresh = fn
}
// SetHubReportPusher sets the callback to push an immediate hub report.
func (w *StorageWatchdog) SetHubReportPusher(fn func()) {
w.pushHubReport = fn
}
// SetRepoUnlocker sets the callback to unlock a restic repo on reconnect.
func (w *StorageWatchdog) SetRepoUnlocker(fn func(ctx context.Context, repoPath string) error) {
w.unlockRepo = fn
}
// Check probes all registered storage paths and reacts to state changes.
// Called by the scheduler every 5 seconds.
func (w *StorageWatchdog) Check(ctx context.Context) error {
paths := w.settings.GetStoragePaths()
if len(paths) == 0 {
return nil
}
for _, sp := range paths {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
state := w.getOrCreateState(sp.Path)
// Rate-limit per-path probes
state.mu.Lock()
if time.Since(state.lastProbeTime) < state.probeInterval {
state.mu.Unlock()
continue
}
state.lastProbeTime = time.Now()
state.mu.Unlock()
// Skip decommissioned drives entirely — no apps reference them
if sp.Decommissioned {
continue
}
// Skip simulated-disconnected paths (handled by debug UI)
if w.isSimulated(sp.Path) {
continue
}
if sp.Disconnected {
w.handleReconnectCheck(ctx, sp)
} else {
w.handleConnectedProbe(sp, state)
}
}
return nil
}
// getOrCreateState returns the in-memory probe state for a path, creating if needed.
func (w *StorageWatchdog) getOrCreateState(path string) *pathProbeState {
w.mu.Lock()
defer w.mu.Unlock()
if s, ok := w.pathState[path]; ok {
return s
}
s := &pathProbeState{
lastStatus: "connected",
probeInterval: defaultProbeInterval,
}
w.pathState[path] = s
return s
}
// handleConnectedProbe probes a connected drive and triggers disconnect if needed.
func (w *StorageWatchdog) handleConnectedProbe(sp settings.StoragePath, state *pathProbeState) {
probeStart := time.Now()
result := system.ProbeStoragePath(sp.Path)
probeLatency := time.Since(probeStart)
state.mu.Lock()
defer state.mu.Unlock()
if w.isDebug() {
state.probeCount++
state.totalLatency += probeLatency
}
if result.Status == system.ProbeConnected {
if state.consecutiveFailures > 0 {
w.logger.Printf("[DEBUG] [storage] Probe recovered for %s after %d failures", sp.Path, state.consecutiveFailures)
}
state.consecutiveFailures = 0
state.lastStatus = "connected"
if w.isDebug() {
state.probeOKCount++
// Every 60 probes (~5 minutes at 5s interval): emit summary
if state.probeCount >= 60 {
avgLatency := state.totalLatency / time.Duration(state.probeCount)
w.logger.Printf("[DEBUG] [storage] Storage watchdog: %s — %d/%d probes OK (last 5m, avg %dms)",
sp.Path, state.probeOKCount, state.probeCount, avgLatency.Milliseconds())
state.probeCount = 0
state.probeOKCount = 0
state.totalLatency = 0
state.lastSummaryTime = time.Now()
}
}
return
}
state.consecutiveFailures++
// Debug: log immediately on unexpected failure (was connected, now failing)
if w.isDebug() && state.lastStatus == "connected" {
w.logger.Printf("[DEBUG] [storage] Storage probe failed for %s (%d/%d before disconnect): %v",
sp.Path, state.consecutiveFailures, probeThreshold, result.Err)
}
w.logger.Printf("[WARN] [storage] Probe failed for %s (%d/%d): %v",
sp.Path, state.consecutiveFailures, probeThreshold, result.Err)
if state.consecutiveFailures >= probeThreshold {
// Release state.mu before calling handleDisconnect (which re-acquires it
// internally). Re-acquire afterwards so the deferred Unlock stays balanced.
// Wrap in a func to guarantee re-lock even if handleDisconnect panics.
func() {
state.mu.Unlock()
defer state.mu.Lock()
w.handleDisconnect(sp, state, result)
}()
}
}
// handleDisconnect reacts to a confirmed drive disconnection.
func (w *StorageWatchdog) handleDisconnect(sp settings.StoragePath, state *pathProbeState, probe system.ProbeResult) {
label := sp.Label
if label == "" {
label = sp.Path
}
w.logger.Printf("[ERROR] [storage] Drive disconnected: %s (%s)", sp.Path, label)
// 1. Find and stop affected stacks
stoppedStacks := w.stopAffectedStacks(sp.Path)
// 2. Mark disconnected in settings (persists to settings.json)
if err := w.settings.SetDisconnected(sp.Path, true, stoppedStacks); err != nil {
w.logger.Printf("[ERROR] [storage] Failed to mark disconnected: %v", err)
}
// 3. Lazy unmount stale mount (if probe timed out — mount is likely hanging)
if probe.Status == system.ProbeTimeout {
w.lazyUnmount(sp.Path)
}
// 4. Update in-memory state
state.mu.Lock()
state.lastStatus = "disconnected"
state.probeInterval = disconnectedProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// 5. Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// 6. Send notification
w.notifier.NotifyStorageDisconnected(label, stoppedStacks)
// 7. Push immediate hub report
if w.pushHubReport != nil {
go w.pushHubReport()
}
}
// handleReconnectCheck checks if a disconnected drive has been reconnected.
func (w *StorageWatchdog) handleReconnectCheck(ctx context.Context, sp settings.StoragePath) {
// Find the UUID for this path from fstab
// For attach-wizard drives, the UUID is on the raw mount, not the bind mount
mountPath := sp.Path
rawPath, isAttachWizard := system.HasFelhomRawMount(hostFstabPath, sp.Path)
if isAttachWizard {
mountPath = rawPath
}
uuid := system.ParseFstabUUID(hostFstabPath, mountPath)
if uuid == "" {
// No UUID in fstab — can't detect reconnection automatically
return
}
if w.isDebug() {
w.logger.Printf("[DEBUG] [storage] Reconnect check for %s: UUID=%s, mountPath=%s, isAttachWizard=%v",
sp.Path, uuid, mountPath, isAttachWizard)
}
// Check if the UUID block device is present
uuidPath := filepath.Join(hostDevUUIDPath, uuid)
if _, err := os.Stat(uuidPath); err != nil {
return // Drive not reconnected yet
}
label := sp.Label
if label == "" {
label = sp.Path
}
w.logger.Printf("[INFO] [storage] Drive reconnected (UUID found), attempting remount: %s (%s)", sp.Path, label)
if w.isDebug() {
w.logger.Printf("[DEBUG] [storage] UUID %s found at %s, mounting %s (raw=%s, attachWizard=%v)",
uuid, uuidPath, sp.Path, rawPath, isAttachWizard)
}
// Attempt remount
if err := w.remount(sp.Path, rawPath, isAttachWizard); err != nil {
w.logger.Printf("[ERROR] [storage] Remount failed for %s: %v", sp.Path, err)
return // Try again next cycle
}
// Verify with a probe
verifyResult := system.ProbeStoragePath(sp.Path)
if verifyResult.Status != system.ProbeConnected {
w.logger.Printf("[ERROR] [storage] Post-remount probe failed for %s: %v", sp.Path, verifyResult.Err)
if w.isDebug() {
w.logger.Printf("[DEBUG] [storage] Post-mount verification failed for %s: status=%v, err=%v",
sp.Path, verifyResult.Status, verifyResult.Err)
}
return
}
if w.isDebug() {
w.logger.Printf("[DEBUG] [storage] Post-mount verification succeeded for %s", sp.Path)
}
w.logger.Printf("[INFO] [storage] Drive successfully remounted: %s (%s)", sp.Path, label)
// Clean stale restic locks
w.cleanResticLocks(ctx, sp.Path)
// Validate stopped stacks — filter to only actually stopped ones
filteredStacks := w.filterStoppedStacks(sp.StoppedStacks)
// Clear disconnected but preserve StoppedStacks for the restart UI
if err := w.settings.SetDisconnected(sp.Path, false, filteredStacks); err != nil {
w.logger.Printf("[ERROR] [storage] Failed to clear disconnected: %v", err)
}
// Update in-memory state
state := w.getOrCreateState(sp.Path)
state.mu.Lock()
state.lastStatus = "connected"
state.probeInterval = defaultProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// Send notification
w.notifier.NotifyStorageReconnected(label)
// Push immediate hub report
if w.pushHubReport != nil {
go w.pushHubReport()
}
}
// stopAffectedStacks stops all deployed stacks whose HDD_PATH matches the disconnected drive.
func (w *StorageWatchdog) stopAffectedStacks(drivePath string) []string {
if w.stackProvider == nil {
return nil
}
var stopped []string
cleanDrive := filepath.Clean(drivePath)
for _, stack := range w.stackProvider.ListDeployedStacks() {
hddPath := w.stackProvider.GetStackHDDPath(stack.Name)
if hddPath == "" {
continue
}
cleanHDD := filepath.Clean(hddPath)
if cleanHDD != cleanDrive && !strings.HasPrefix(cleanHDD, cleanDrive+"/") {
continue
}
// Don't stop protected stacks
if w.cfg.IsProtectedStack(stack.Name) {
w.logger.Printf("[WARN] [storage] Skipping protected stack: %s", stack.Name)
continue
}
w.logger.Printf("[INFO] [storage] Stopping stack %s (drive disconnected: %s)", stack.Name, drivePath)
if err := w.stackProvider.StopStack(stack.Name); err != nil {
w.logger.Printf("[ERROR] [storage] Failed to stop stack %s: %v", stack.Name, err)
continue // Don't add to stopped list if stop failed
}
stopped = append(stopped, stack.Name)
}
if len(stopped) > 0 {
w.logger.Printf("[INFO] [storage] Stopped %d stack(s) due to drive disconnect: %v", len(stopped), stopped)
}
return stopped
}
// lazyUnmount performs a lazy unmount of a path and its raw mount (if attach-wizard).
func (w *StorageWatchdog) lazyUnmount(path string) {
// For attach-wizard, unmount bind first, then raw
rawPath, isAttachWizard := system.HasFelhomRawMount(hostFstabPath, path)
// Unmount the bind/main path
cmd := exec.Command("umount", "-l", path)
if out, err := cmd.CombinedOutput(); err != nil {
w.logger.Printf("[WARN] [storage] umount -l %s: %v (%s)", path, err, strings.TrimSpace(string(out)))
} else {
w.logger.Printf("[INFO] [storage] Lazy unmounted: %s", path)
}
// Then unmount the raw path if it's an attach-wizard drive
if isAttachWizard && rawPath != "" {
cmd = exec.Command("umount", "-l", rawPath)
if out, err := cmd.CombinedOutput(); err != nil {
w.logger.Printf("[WARN] [storage] umount -l %s: %v (%s)", rawPath, err, strings.TrimSpace(string(out)))
} else {
w.logger.Printf("[INFO] [storage] Lazy unmounted raw: %s", rawPath)
}
}
}
// remount attempts to remount a storage path using fstab entries.
func (w *StorageWatchdog) remount(path, rawPath string, isAttachWizard bool) error {
// Clean any stale mount entries first
exec.Command("umount", "-l", path).Run()
if isAttachWizard && rawPath != "" {
exec.Command("umount", "-l", rawPath).Run()
}
if isAttachWizard && rawPath != "" {
// Mount raw first, then bind
cmd := exec.Command("mount", "-T", hostFstabPath, rawPath)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount raw %s: %v (%s)", rawPath, err, strings.TrimSpace(string(out)))
}
w.logger.Printf("[INFO] [storage] Mounted raw: %s", rawPath)
cmd = exec.Command("mount", "-T", hostFstabPath, path)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount bind %s: %v (%s)", path, err, strings.TrimSpace(string(out)))
}
w.logger.Printf("[INFO] [storage] Mounted bind: %s", path)
} else {
cmd := exec.Command("mount", "-T", hostFstabPath, path)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mount %s: %v (%s)", path, err, strings.TrimSpace(string(out)))
}
w.logger.Printf("[INFO] [storage] Mounted: %s", path)
}
return nil
}
// cleanResticLocks runs restic unlock on the primary repo for a drive path.
func (w *StorageWatchdog) cleanResticLocks(ctx context.Context, drivePath string) {
repoPath := filepath.Join(drivePath, primaryResticSubpath)
locksDir := filepath.Join(repoPath, "locks")
entries, err := os.ReadDir(locksDir)
if err != nil || len(entries) == 0 {
return // No locks dir or no lock files
}
w.logger.Printf("[INFO] [storage] Found %d restic lock file(s) in %s, running unlock", len(entries), repoPath)
if w.unlockRepo != nil {
if err := w.unlockRepo(ctx, repoPath); err != nil {
w.logger.Printf("[WARN] [storage] Restic unlock failed for %s: %v", repoPath, err)
}
}
}
// filterStoppedStacks validates that stacks in the list still exist as deployed stacks.
func (w *StorageWatchdog) filterStoppedStacks(stackNames []string) []string {
if w.stackProvider == nil || len(stackNames) == 0 {
return nil
}
deployed := make(map[string]bool)
for _, s := range w.stackProvider.ListDeployedStacks() {
deployed[s.Name] = true
}
var result []string
for _, name := range stackNames {
if deployed[name] {
result = append(result, name)
}
}
return result
}
// SafeDisconnect performs a safe disconnect of a storage path.
// Stops affected apps, syncs filesystem, and unmounts the drive.
func (w *StorageWatchdog) SafeDisconnect(ctx context.Context, path string) (stoppedStacks []string, err error) {
sp := w.findStoragePath(path)
if sp == nil {
return nil, fmt.Errorf("storage path %q not found", path)
}
if sp.Disconnected {
return nil, fmt.Errorf("drive already disconnected")
}
if sp.Decommissioned {
return nil, fmt.Errorf("drive is decommissioned — no apps to stop")
}
label := sp.Label
if label == "" {
label = sp.Path
}
w.logger.Printf("[INFO] [storage] Safe disconnect requested: %s (%s)", path, label)
// 1. Stop affected stacks
stoppedStacks = w.stopAffectedStacks(path)
// 2. Sync filesystem
exec.Command("sync").Run()
// 3. Unmount
rawPath, isAttachWizard := system.HasFelhomRawMount(hostFstabPath, path)
// Unmount bind/main
cmd := exec.Command("umount", path)
if out, umountErr := cmd.CombinedOutput(); umountErr != nil {
// Try lazy unmount as fallback
w.logger.Printf("[WARN] [storage] umount %s failed, trying lazy: %v", path, umountErr)
cmd = exec.Command("umount", "-l", path)
if out, umountErr = cmd.CombinedOutput(); umountErr != nil {
return stoppedStacks, fmt.Errorf("umount %s failed: %v (%s)", path, umountErr, strings.TrimSpace(string(out)))
}
}
// Unmount raw if attach-wizard
if isAttachWizard && rawPath != "" {
cmd = exec.Command("umount", rawPath)
if out, umountErr := cmd.CombinedOutput(); umountErr != nil {
cmd = exec.Command("umount", "-l", rawPath)
if out, umountErr = cmd.CombinedOutput(); umountErr != nil {
w.logger.Printf("[WARN] [storage] umount raw %s failed: %v (%s)", rawPath, umountErr, strings.TrimSpace(string(out)))
}
}
}
// 4. Mark disconnected
if setErr := w.settings.SetDisconnected(path, true, stoppedStacks); setErr != nil {
w.logger.Printf("[ERROR] [storage] Failed to mark disconnected: %v", setErr)
}
// 5. Update in-memory state
state := w.getOrCreateState(path)
state.mu.Lock()
state.lastStatus = "disconnected"
state.probeInterval = disconnectedProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// 6. Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// 7. Notify and push hub report
w.notifier.Notify("storage_safe_disconnect", "info",
fmt.Sprintf("Meghajtó biztonságosan leválasztva: %s", label), "")
if w.pushHubReport != nil {
go w.pushHubReport()
}
w.logger.Printf("[INFO] [storage] Safe disconnect completed: %s — drive can be removed", path)
return stoppedStacks, nil
}
// Reconnect attempts to remount a disconnected storage path.
func (w *StorageWatchdog) Reconnect(ctx context.Context, path string) (stoppedStacks []string, err error) {
sp := w.findStoragePath(path)
if sp == nil {
return nil, fmt.Errorf("storage path %q not found", path)
}
if !sp.Disconnected {
return nil, fmt.Errorf("drive is not disconnected")
}
label := sp.Label
if label == "" {
label = sp.Path
}
// Check UUID availability
mountPath := sp.Path
rawPath, isAttachWizard := system.HasFelhomRawMount(hostFstabPath, sp.Path)
if isAttachWizard {
mountPath = rawPath
}
uuid := system.ParseFstabUUID(hostFstabPath, mountPath)
if uuid != "" {
uuidPath := filepath.Join(hostDevUUIDPath, uuid)
if _, statErr := os.Stat(uuidPath); statErr != nil {
return nil, fmt.Errorf("drive not detected (UUID %s not found) — ensure the drive is physically connected", uuid)
}
}
// Attempt remount
if mountErr := w.remount(path, rawPath, isAttachWizard); mountErr != nil {
return nil, fmt.Errorf("mount failed: %w", mountErr)
}
// Verify
verifyResult := system.ProbeStoragePath(path)
if verifyResult.Status != system.ProbeConnected {
return nil, fmt.Errorf("mount appeared to succeed but probe failed: %v", verifyResult.Err)
}
// Clean restic locks
w.cleanResticLocks(ctx, path)
// Validate stopped stacks
filteredStacks := w.filterStoppedStacks(sp.StoppedStacks)
// Clear disconnected, preserve stopped stacks for restart UI
if setErr := w.settings.SetDisconnected(path, false, filteredStacks); setErr != nil {
w.logger.Printf("[ERROR] [storage] Failed to clear disconnected: %v", setErr)
}
// Update in-memory state
state := w.getOrCreateState(path)
state.mu.Lock()
state.lastStatus = "connected"
state.probeInterval = defaultProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// Notify
w.notifier.NotifyStorageReconnected(label)
if w.pushHubReport != nil {
go w.pushHubReport()
}
w.logger.Printf("[INFO] [storage] Reconnect completed: %s", path)
return filteredStacks, nil
}
// RestartStoppedApps restarts apps that were auto-stopped due to a drive disconnect.
func (w *StorageWatchdog) RestartStoppedApps(path string) (started, failed []string) {
sp := w.findStoragePath(path)
if sp == nil || sp.Disconnected {
return nil, nil
}
stacks := w.settings.GetStoppedStacks(path)
if len(stacks) == 0 {
return nil, nil
}
for _, name := range stacks {
w.logger.Printf("[INFO] [storage] Starting stack %s (drive reconnected: %s)", name, path)
if err := w.stackProvider.StartStack(name); err != nil {
w.logger.Printf("[ERROR] [storage] Failed to start stack %s: %v", name, err)
failed = append(failed, name)
} else {
started = append(started, name)
}
}
// Clear stopped stacks list
if err := w.settings.ClearStoppedStacks(path); err != nil {
w.logger.Printf("[ERROR] [storage] Failed to clear stopped stacks: %v", err)
}
return started, failed
}
// ── Debug simulation methods ─────────────────────────────────────────
// isSimulated returns true if the path is in simulated-disconnect state.
func (w *StorageWatchdog) isSimulated(path string) bool {
w.simulatedMu.RLock()
defer w.simulatedMu.RUnlock()
return w.simulatedPaths[path]
}
// SimulateDisconnect simulates a drive disconnection without actually unmounting.
// Runs disconnect steps 1,2,4,5,6,7 (skips step 3: lazyUnmount).
// Returns the list of stopped stacks.
func (w *StorageWatchdog) SimulateDisconnect(ctx context.Context, path string) ([]string, error) {
sp := w.findStoragePath(path)
if sp == nil {
return nil, fmt.Errorf("storage path %q not found", path)
}
if sp.Disconnected {
return nil, fmt.Errorf("drive already disconnected")
}
if sp.Decommissioned {
return nil, fmt.Errorf("drive is decommissioned")
}
label := sp.Label
if label == "" {
label = sp.Path
}
w.logger.Printf("[INFO] [storage] (simulation) Simulating disconnect: %s (%s)", path, label)
// Mark as simulated so the watchdog skips probing this path
w.simulatedMu.Lock()
w.simulatedPaths[path] = true
w.simulatedMu.Unlock()
// Step 1: Stop affected stacks
stoppedStacks := w.stopAffectedStacks(path)
// Step 2: Mark disconnected in settings
if err := w.settings.SetDisconnected(path, true, stoppedStacks); err != nil {
w.logger.Printf("[ERROR] [storage] (simulation) Failed to mark disconnected: %v", err)
}
// Step 3: SKIPPED (no lazyUnmount — drive stays physically mounted)
// Step 4: Update in-memory state
state := w.getOrCreateState(path)
state.mu.Lock()
state.lastStatus = "disconnected"
state.probeInterval = disconnectedProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// Step 5: Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// Step 6: Send notification
w.notifier.NotifyStorageDisconnected(label, stoppedStacks)
// Step 7: Push hub report
if w.pushHubReport != nil {
go w.pushHubReport()
}
w.logger.Printf("[INFO] [storage] (simulation) Disconnect simulated: %s — %d stack(s) stopped", path, len(stoppedStacks))
return stoppedStacks, nil
}
// SimulateReconnect undoes a simulated disconnection.
func (w *StorageWatchdog) SimulateReconnect(ctx context.Context, path string) error {
if !w.isSimulated(path) {
return fmt.Errorf("path %q is not in simulated-disconnect state", path)
}
sp := w.findStoragePath(path)
if sp == nil {
return fmt.Errorf("storage path %q not found", path)
}
label := sp.Label
if label == "" {
label = sp.Path
}
w.logger.Printf("[INFO] [storage] (simulation) Simulating reconnect: %s (%s)", path, label)
// Remove from simulated set
w.simulatedMu.Lock()
delete(w.simulatedPaths, path)
w.simulatedMu.Unlock()
// Verify drive is actually still mounted (it should be since we never unmounted)
verifyResult := system.ProbeStoragePath(path)
if verifyResult.Status != system.ProbeConnected {
return fmt.Errorf("drive probe failed after simulation clear: %v", verifyResult.Err)
}
// Clean restic locks
w.cleanResticLocks(ctx, path)
// Validate stopped stacks
filteredStacks := w.filterStoppedStacks(sp.StoppedStacks)
// Clear disconnected, preserve stopped stacks for restart UI
if err := w.settings.SetDisconnected(path, false, filteredStacks); err != nil {
w.logger.Printf("[ERROR] [storage] (simulation) Failed to clear disconnected: %v", err)
}
// Update in-memory state
state := w.getOrCreateState(path)
state.mu.Lock()
state.lastStatus = "connected"
state.probeInterval = defaultProbeInterval
state.consecutiveFailures = 0
state.mu.Unlock()
// Trigger alert refresh
if w.alertRefresh != nil {
w.alertRefresh()
}
// Send notification
w.notifier.NotifyStorageReconnected(label)
if w.pushHubReport != nil {
go w.pushHubReport()
}
w.logger.Printf("[INFO] [storage] (simulation) Reconnect simulated: %s", path)
return nil
}
// PathDebugStatus holds per-path probe state for the debug page.
type PathDebugStatus struct {
Path string `json:"path"`
Label string `json:"label"`
Status string `json:"status"`
Simulated bool `json:"simulated"`
ProbeOK bool `json:"probe_ok"`
DebounceCount int `json:"debounce_count"`
DebounceMax int `json:"debounce_max"`
LastProbe time.Time `json:"last_probe"`
AvgLatencyMs float64 `json:"avg_latency_ms"`
ProbeCount int `json:"probe_count"`
ProbeOKCount int `json:"probe_ok_count"`
}
// GetDebugStatus returns per-path probe state for the debug page.
func (w *StorageWatchdog) GetDebugStatus() []PathDebugStatus {
paths := w.settings.GetStoragePaths()
result := make([]PathDebugStatus, 0, len(paths))
w.mu.Lock()
defer w.mu.Unlock()
for _, sp := range paths {
if sp.Decommissioned {
continue
}
ds := PathDebugStatus{
Path: sp.Path,
Label: sp.Label,
DebounceMax: probeThreshold,
}
if sp.Disconnected {
ds.Status = "disconnected"
} else {
ds.Status = "connected"
}
ds.Simulated = w.isSimulatedLocked(sp.Path)
if state, ok := w.pathState[sp.Path]; ok {
state.mu.Lock()
ds.DebounceCount = state.consecutiveFailures
ds.LastProbe = state.lastProbeTime
ds.ProbeOK = state.lastStatus == "connected"
ds.ProbeCount = state.probeCount
ds.ProbeOKCount = state.probeOKCount
if state.probeCount > 0 {
ds.AvgLatencyMs = float64(state.totalLatency.Milliseconds()) / float64(state.probeCount)
}
state.mu.Unlock()
}
result = append(result, ds)
}
return result
}
// isSimulatedLocked checks simulation state without acquiring simulatedMu
// (caller must hold w.mu or be ok with a racy read for debug display).
func (w *StorageWatchdog) isSimulatedLocked(path string) bool {
w.simulatedMu.RLock()
defer w.simulatedMu.RUnlock()
return w.simulatedPaths[path]
}
// findStoragePath returns the storage path entry for a given path, or nil.
func (w *StorageWatchdog) findStoragePath(path string) *settings.StoragePath {
for _, sp := range w.settings.GetStoragePaths() {
if sp.Path == path {
return &sp
}
}
return nil
}
controller/internal/report/builder.go
+4 -50
View File
@@ -6,8 +6,6 @@ import (
"fmt"
"log"
"os"
"strconv"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
@@ -241,32 +239,16 @@ func buildBackupReport(cfg *config.Config, backupMgr *backup.Manager) BackupRepo
return br
}
// Disk-tier backup (restic snapshots, integrity check, repo stats) has moved to
// the host agent (slice 8C). The controller report now covers only app-data backup
// (database dumps); restic/snapshot/integrity fields are left zero.
nextDBDump := scheduler.NextDailyRun(cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(cfg.Backup.ResticSchedule)
status := backupMgr.GetFullStatus(nextDBDump, nextBackup)
status := backupMgr.GetFullStatus(nextDBDump)
if status.LastDBDump != nil {
t := status.LastDBDump.LastRun
br.LastDBDump = &t
}
if status.LastBackup != nil {
t := status.LastBackup.LastRun
br.LastSnapshot = &t
}
if status.RepoStats != nil {
br.SnapshotCount = status.RepoStats.SnapshotCount
br.RepoSizeMB = parseSizeToMB(status.RepoStats.TotalSize)
}
if !status.LastCheckTime.IsZero() {
t := status.LastCheckTime
br.LastIntegrityCheck = &t
}
br.IntegrityOK = status.LastCheckOK
// Include restic password for hub-side disaster recovery
if pw, err := backupMgr.GetResticPassword(); err == nil {
br.ResticPassword = pw
}
return br
}
@@ -296,31 +278,3 @@ func buildStacksReport(stackMgr *stacks.Manager) StacksReport {
return sr
}
// parseSizeToMB parses a formatted size string like "1.5 GB", "512.0 MB" into MB.
func parseSizeToMB(s string) int64 {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
parts := strings.Fields(s)
if len(parts) != 2 {
return 0
}
val, err := strconv.ParseFloat(parts[0], 64)
if err != nil {
return 0
}
switch strings.ToUpper(parts[1]) {
case "GB":
return int64(val * 1024)
case "MB":
return int64(val)
case "KB":
return int64(val / 1024)
default:
return int64(val)
}
}
controller/internal/report/config_pull.go
+61
View File
@@ -0,0 +1,61 @@
package report
import (
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// Config-pull error types for setup-wizard UI display.
//
// These (and PullConfig) were previously part of infra_pull.go alongside the
// disk-tier DR-recovery (PullRecovery / infra-backup) client. Disk recovery moved
// to the host agent in slice 8C; only the fresh-install config download survives
// here, so it lives in this slimmed file.
var (
ErrHubUnreachable = errors.New("hub unreachable")
ErrAuthFailed = errors.New("authentication failed")
ErrNotFound = errors.New("customer not found")
ErrHubError = errors.New("hub error")
)
// PullConfig fetches a generated controller.yaml from the Hub config endpoint.
// Auth: X-Retrieval-Password header. Used by the setup wizard's fresh-install flow.
func PullConfig(hubURL, customerID, retrievalPassword string) (string, error) {
url := strings.TrimRight(hubURL, "/") + "/api/v1/config/" + customerID
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", fmt.Errorf("%w: %v", ErrHubError, err)
}
req.Header.Set("X-Retrieval-Password", retrievalPassword)
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("%w: %v", ErrHubUnreachable, err)
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
// success
case http.StatusUnauthorized:
return "", ErrAuthFailed
case http.StatusNotFound:
return "", ErrNotFound
default:
return "", fmt.Errorf("%w: HTTP %d", ErrHubError, resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) // 1MB limit
if err != nil {
return "", fmt.Errorf("%w: reading response: %v", ErrHubError, err)
}
return string(body), nil
}
controller/internal/report/infra_backup.go
-129
View File
@@ -1,129 +0,0 @@
package report
import (
"encoding/base64"
"fmt"
"log"
"os"
"path/filepath"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// InfraBackup is the payload pushed to the Hub for disaster recovery.
type InfraBackup struct {
CustomerID string `json:"customer_id"`
Domain string `json:"domain"`
ControllerVersion string `json:"controller_version"`
Timestamp string `json:"timestamp"`
ControllerConfigB64 string `json:"controller_config_b64"`
SettingsJSONB64 string `json:"settings_json_b64,omitempty"`
DiskLayout backup.DiskLayout `json:"disk_layout"`
DeployedStacks []InfraStack `json:"deployed_stacks"`
ResticPassword string `json:"restic_password,omitempty"`
CrossDrivePassword string `json:"cross_drive_password,omitempty"`
EncryptionKeyB64 string `json:"encryption_key_b64,omitempty"`
}
// InfraStack identifies a deployed app for disaster recovery.
// Note: AppYamlB64 contains encrypted secrets (ENC:... values).
// The encryption key is also in this backup (EncryptionKeyB64).
// This is intentional — the infra backup must be self-contained for DR.
// Physical security of the backup media protects both.
type InfraStack struct {
Name string `json:"name"`
DisplayName string `json:"display_name"`
HDDPath string `json:"hdd_path,omitempty"`
NeedsHDD bool `json:"needs_hdd"`
DockerComposeB64 string `json:"docker_compose_b64,omitempty"`
AppYamlB64 string `json:"app_yaml_b64,omitempty"`
FelhomYamlB64 string `json:"felhom_yaml_b64,omitempty"`
}
// BuildInfraBackup collects all infrastructure state for Hub backup.
func BuildInfraBackup(
customerID, domain, version string,
controllerYAMLPath string,
settingsPath string,
resticPasswordFile string,
encryptionKeyFile string,
systemDataPath string,
sett *settings.Settings,
stackProvider backup.StackDataProvider,
logger *log.Logger,
) (*InfraBackup, error) {
ib := &InfraBackup{
CustomerID: customerID,
Domain: domain,
ControllerVersion: version,
Timestamp: time.Now().UTC().Format(time.RFC3339),
}
// Read and encode controller.yaml (critical — fail if unreadable)
data, err := os.ReadFile(controllerYAMLPath)
if err != nil {
return nil, fmt.Errorf("reading controller config %s: %w", controllerYAMLPath, err)
}
ib.ControllerConfigB64 = base64.StdEncoding.EncodeToString(data)
// Read and encode settings.json (important but non-fatal)
if data, err := os.ReadFile(settingsPath); err == nil {
ib.SettingsJSONB64 = base64.StdEncoding.EncodeToString(data)
} else if !os.IsNotExist(err) {
logger.Printf("[WARN] [report] Infra backup: could not read settings.json: %v", err)
}
// Read primary restic password (important but non-fatal)
if data, err := os.ReadFile(resticPasswordFile); err == nil {
ib.ResticPassword = base64.StdEncoding.EncodeToString(data)
} else if !os.IsNotExist(err) {
logger.Printf("[WARN] [report] Infra backup: could not read restic password file: %v", err)
}
// Read encryption key for app.yaml secrets (important but non-fatal)
if encryptionKeyFile != "" {
if data, err := os.ReadFile(encryptionKeyFile); err == nil {
ib.EncryptionKeyB64 = base64.StdEncoding.EncodeToString(data)
} else if !os.IsNotExist(err) {
logger.Printf("[WARN] [report] Infra backup: could not read encryption key file: %v", err)
}
}
// Collect disk layout from fstab + blkid
ib.DiskLayout = collectDiskLayout(systemDataPath)
// Collect deployed stacks (including actual config files for DR)
deployed := stackProvider.ListDeployedStacks()
for _, s := range deployed {
is := InfraStack{
Name: s.Name,
DisplayName: s.DisplayName,
HDDPath: stackProvider.GetStackHDDPath(s.Name),
NeedsHDD: s.NeedsHDD,
}
if composePath, ok := stackProvider.GetStackComposePath(s.Name); ok {
stackDir := filepath.Dir(composePath)
if data, err := os.ReadFile(filepath.Join(stackDir, "docker-compose.yml")); err == nil {
is.DockerComposeB64 = base64.StdEncoding.EncodeToString(data)
}
if data, err := os.ReadFile(filepath.Join(stackDir, "app.yaml")); err == nil {
is.AppYamlB64 = base64.StdEncoding.EncodeToString(data)
}
if data, err := os.ReadFile(filepath.Join(stackDir, ".felhom.yml")); err == nil {
is.FelhomYamlB64 = base64.StdEncoding.EncodeToString(data)
}
}
ib.DeployedStacks = append(ib.DeployedStacks, is)
}
if ib.DeployedStacks == nil {
ib.DeployedStacks = []InfraStack{}
}
logger.Printf("[INFO] [report] InfraBackup built successfully (stacks=%d)", len(ib.DeployedStacks))
return ib, nil
}
controller/internal/report/infra_backup_linux.go
-135
View File
@@ -1,135 +0,0 @@
//go:build linux
package report
import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
)
// collectDiskLayout reads /host-fstab and correlates with blkid/lsblk to build
// the disk mount topology. Only includes data partitions (not root, boot, or swap).
func collectDiskLayout(systemDataPath string) backup.DiskLayout {
layout := backup.DiskLayout{}
fstabPath := "/host-fstab"
if _, err := os.Stat(fstabPath); err != nil {
fstabPath = "/etc/fstab"
}
data, err := os.ReadFile(fstabPath)
if err != nil {
return layout
}
// Parse fstab into UUID-based entries and bind mount entries
type fstabEntry struct {
source string
mountPoint string
fsType string
options string
}
var uuidEntries []fstabEntry
var bindEntries []fstabEntry
systemMounts := map[string]bool{"/": true, "/boot": true, "/boot/efi": true}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
source := fields[0]
mountPoint := fields[1]
fsType := fields[2]
options := fields[3]
// Skip system mounts and swap
if systemMounts[mountPoint] || fsType == "swap" {
continue
}
if strings.HasPrefix(source, "UUID=") {
uuidEntries = append(uuidEntries, fstabEntry{
source: strings.TrimPrefix(source, "UUID="),
mountPoint: mountPoint,
fsType: fsType,
options: options,
})
} else if fsType == "none" && strings.Contains(options, "bind") {
bindEntries = append(bindEntries, fstabEntry{
source: source,
mountPoint: mountPoint,
options: options,
})
}
}
// Process UUID-based entries
for _, e := range uuidEntries {
dm := backup.DiskMount{
UUID: e.source,
MountPoint: e.mountPoint,
FSType: e.fsType,
FstabOptions: e.options,
}
// Get label via blkid
if out, err := exec.Command("blkid", "-o", "value", "-s", "LABEL", "-U", e.source).Output(); err == nil {
dm.Label = strings.TrimSpace(string(out))
}
// Get size via lsblk (resolve UUID to device first)
if devPath, err := exec.Command("blkid", "-U", e.source).Output(); err == nil {
dev := strings.TrimSpace(string(devPath))
if dev != "" {
if out, err := exec.Command("lsblk", "-b", "-n", "-o", "SIZE", dev).Output(); err == nil {
if sz, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64); err == nil {
dm.SizeBytes = sz
}
}
}
}
// Determine role
if e.mountPoint == systemDataPath {
dm.Role = "system_data"
} else {
dm.Role = "hdd_storage"
}
// Check for a corresponding bind mount
for _, bind := range bindEntries {
if strings.HasPrefix(bind.source, e.mountPoint+"/") {
subdir := strings.TrimPrefix(bind.source, e.mountPoint+"/")
dm.BindSubdir = subdir
dm.RawMount = e.mountPoint
dm.MountPoint = bind.mountPoint // the final user-facing mount point
break
}
}
// Get label from mount point basename as fallback
if dm.Label == "" {
if dm.RawMount != "" {
dm.Label = filepath.Base(dm.RawMount)
} else {
dm.Label = filepath.Base(dm.MountPoint)
}
}
layout.Mounts = append(layout.Mounts, dm)
}
return layout
}
controller/internal/report/infra_backup_other.go
-11
View File
@@ -1,11 +0,0 @@
//go:build !linux
package report
import "gitea.dooplex.hu/admin/felhom-controller/internal/backup"
// collectDiskLayout is a no-op on non-Linux platforms.
// The controller only runs on Linux; this stub allows cross-compilation.
func collectDiskLayout(systemDataPath string) backup.DiskLayout {
return backup.DiskLayout{}
}
controller/internal/report/infra_pull.go
-201
View File
@@ -1,201 +0,0 @@
package report
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// Recovery pull error types for UI display.
var (
ErrHubUnreachable = errors.New("hub unreachable")
ErrAuthFailed = errors.New("authentication failed")
ErrNotFound = errors.New("customer not found")
ErrHubError = errors.New("hub error")
)
// BackupVersionSummary holds metadata about one backup version (from Hub).
type BackupVersionSummary struct {
ID int64 `json:"id"`
CreatedAt string `json:"created_at"`
StackCount int `json:"stack_count"`
DiskCount int `json:"disk_count"`
StackNames []string `json:"stack_names,omitempty"`
}
// RecoveryResponse is the combined config + infra backup from the Hub recovery endpoint.
type RecoveryResponse struct {
CustomerID string `json:"customer_id"`
ConfigYAML string `json:"config_yaml"`
InfraBackup *InfraBackup `json:"infra_backup"`
HasInfraBackup bool `json:"has_infra_backup"`
BackupVersions []BackupVersionSummary `json:"backup_versions,omitempty"`
}
// PullRecovery fetches combined recovery data from the Hub (config + infra backup).
// Auth: X-Retrieval-Password header.
func PullRecovery(hubURL, customerID, retrievalPassword string) (*RecoveryResponse, error) {
url := strings.TrimRight(hubURL, "/") + "/api/v1/recovery/" + customerID
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrHubError, err)
}
req.Header.Set("X-Retrieval-Password", retrievalPassword)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrHubUnreachable, err)
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
// success, continue below
case http.StatusUnauthorized:
return nil, ErrAuthFailed
case http.StatusNotFound:
return nil, ErrNotFound
default:
return nil, fmt.Errorf("%w: HTTP %d", ErrHubError, resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20)) // 10MB limit
if err != nil {
return nil, fmt.Errorf("%w: reading response: %v", ErrHubError, err)
}
var rr RecoveryResponse
if err := json.Unmarshal(body, &rr); err != nil {
return nil, fmt.Errorf("%w: parsing response: %v", ErrHubError, err)
}
return &rr, nil
}
// PullRecoveryVersion fetches recovery data for a specific backup version ID.
func PullRecoveryVersion(hubURL, customerID, retrievalPassword string, versionID int64) (*RecoveryResponse, error) {
url := strings.TrimRight(hubURL, "/") + "/api/v1/recovery/" + customerID + fmt.Sprintf("?version=%d", versionID)
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrHubError, err)
}
req.Header.Set("X-Retrieval-Password", retrievalPassword)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrHubUnreachable, err)
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
// success
case http.StatusUnauthorized:
return nil, ErrAuthFailed
case http.StatusNotFound:
return nil, ErrNotFound
default:
return nil, fmt.Errorf("%w: HTTP %d", ErrHubError, resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20))
if err != nil {
return nil, fmt.Errorf("%w: reading response: %v", ErrHubError, err)
}
var rr RecoveryResponse
if err := json.Unmarshal(body, &rr); err != nil {
return nil, fmt.Errorf("%w: parsing response: %v", ErrHubError, err)
}
return &rr, nil
}
// PullConfig fetches a generated controller.yaml from the Hub config endpoint.
// Auth: X-Retrieval-Password header.
func PullConfig(hubURL, customerID, retrievalPassword string) (string, error) {
url := strings.TrimRight(hubURL, "/") + "/api/v1/config/" + customerID
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", fmt.Errorf("%w: %v", ErrHubError, err)
}
req.Header.Set("X-Retrieval-Password", retrievalPassword)
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("%w: %v", ErrHubUnreachable, err)
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
// success
case http.StatusUnauthorized:
return "", ErrAuthFailed
case http.StatusNotFound:
return "", ErrNotFound
default:
return "", fmt.Errorf("%w: HTTP %d", ErrHubError, resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) // 1MB limit
if err != nil {
return "", fmt.Errorf("%w: reading response: %v", ErrHubError, err)
}
return string(body), nil
}
// PullInfraBackup fetches the infrastructure backup from the Hub.
// Returns nil, nil if no backup exists for this customer.
func PullInfraBackup(hubURL, apiKey, customerID string) (*InfraBackup, error) {
url := strings.TrimRight(hubURL, "/") + "/api/v1/infra-backup/" + customerID
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
if apiKey != "" {
req.Header.Set("Authorization", "Bearer "+apiKey)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("hub request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil // no backup for this customer
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB limit
if err != nil {
return nil, fmt.Errorf("reading response: %w", err)
}
var ib InfraBackup
if err := json.Unmarshal(body, &ib); err != nil {
return nil, fmt.Errorf("parsing infra backup: %w", err)
}
return &ib, nil
}
controller/internal/selftest/selftest.go
-32
View File
@@ -11,7 +11,6 @@ import (
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
@@ -45,7 +44,6 @@ func Run(cfg *config.Config, sett *settings.Settings, logger *log.Logger) *Summa
func() CheckResult { return checkStoragePaths(sett) },
func() CheckResult { return checkGitCatalog(cfg.Paths.StacksDir) },
func() CheckResult { return checkHubConnectivity(cfg) },
func() CheckResult { return checkResticRepos(sett) },
func() CheckResult { return checkMetricsDB(cfg.Paths.DataDir) },
}
@@ -214,36 +212,6 @@ func checkHubConnectivity(cfg *config.Config) CheckResult {
return CheckResult{Name: "Hub connectivity", Status: "warn", Message: fmt.Sprintf("HTTP %d from %s", resp.StatusCode, url)}
}
func checkResticRepos(sett *settings.Settings) CheckResult {
paths := sett.GetStoragePaths()
if len(paths) == 0 {
return CheckResult{Name: "Restic repos", Status: "pass", Message: "no storage paths, skipped"}
}
found := 0
missing := 0
for _, sp := range paths {
if sp.Disconnected || sp.Decommissioned {
continue
}
repoPath := backup.PrimaryResticRepoPath(sp.Path)
if _, err := os.Stat(repoPath); err == nil {
found++
} else {
missing++
}
}
if found == 0 && missing > 0 {
return CheckResult{Name: "Restic repos", Status: "warn", Message: fmt.Sprintf("0 repos found, %d expected", missing)}
}
msg := fmt.Sprintf("%d repos found", found)
if missing > 0 {
msg += fmt.Sprintf(", %d missing", missing)
}
return CheckResult{Name: "Restic repos", Status: "pass", Message: msg}
}
func checkMetricsDB(dataDir string) CheckResult {
dbPath := filepath.Join(dataDir, "metrics.db")
info, err := os.Stat(dbPath)
controller/internal/setup/handlers.go
+2 -586
View File
@@ -1,13 +1,10 @@
package setup
import (
"context"
crand "crypto/rand"
"crypto/sha256"
"embed"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"html/template"
"log"
@@ -15,10 +12,8 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
@@ -38,27 +33,6 @@ type Server struct {
tmpl *template.Template
state *SetupState
version string
// Scan state for async drive scanning
scanMu sync.Mutex
scanRunning bool
scanResults []DriveBackup
scanDone bool
scanError string
// Restore progress
restoreMu sync.Mutex
restoreRunning bool
restoreSteps []RestoreStep
restoreError string
restoreDone bool
}
// RestoreStep tracks progress of a restore operation.
type RestoreStep struct {
Label string `json:"label"`
Status string `json:"status"` // "pending", "running", "done", "failed"
Error string `json:"error,omitempty"`
}
// NewServer creates a new setup wizard server.
@@ -111,14 +85,10 @@ func (s *Server) loadTemplates() {
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
// Disk-recovery setup paths (drive scan, infra-backup restore) have moved to the
// host agent (slice 8C). The wizard now offers fresh install (from Hub) + manual.
mux.HandleFunc("/", s.handleRoot)
mux.HandleFunc("/setup", s.handleWelcome)
mux.HandleFunc("/setup/scan", s.handleScan)
mux.HandleFunc("/setup/scan/status", s.handleScanStatus)
mux.HandleFunc("/setup/hub-restore", s.handleHubRestore)
mux.HandleFunc("/setup/hub-restore/select", s.handleHubVersionSelect)
mux.HandleFunc("/setup/restore", s.handleRestore)
mux.HandleFunc("/setup/restore/status", s.handleRestoreStatus)
mux.HandleFunc("/setup/fresh", s.handleFreshHub)
mux.HandleFunc("/setup/manual", s.handleManual)
mux.HandleFunc("/setup/failed", s.handleFailed)
@@ -161,110 +131,6 @@ func (s *Server) handleWelcome(w http.ResponseWriter, r *http.Request) {
s.render(w, "setup_welcome", data)
}
func (s *Server) handleScan(w http.ResponseWriter, r *http.Request) {
csrf := ensureCSRFToken(w, r)
// Start scan if not already running
s.scanMu.Lock()
if !s.scanRunning && !s.scanDone {
s.scanRunning = true
go s.runDriveScan()
}
s.scanMu.Unlock()
s.state.SetStep("scan")
data := map[string]interface{}{
"CSRF": csrf,
}
s.render(w, "setup_scan", data)
}
func (s *Server) handleScanStatus(w http.ResponseWriter, r *http.Request) {
s.scanMu.Lock()
defer s.scanMu.Unlock()
resp := map[string]interface{}{
"running": s.scanRunning,
"done": s.scanDone,
"results": s.scanResults,
"error": s.scanError,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func (s *Server) handleHubRestore(w http.ResponseWriter, r *http.Request) {
csrf := ensureCSRFToken(w, r)
if r.Method == http.MethodPost {
if !validateCSRF(r) {
http.Error(w, "Invalid CSRF token", http.StatusForbidden)
return
}
s.processHubRestore(w, r)
return
}
// Auto-process if credentials are pre-seeded (hub mode from docker-setup.sh)
if s.isHubPreseeded() {
customerID := s.state.GetFormField("customer_id")
password := s.state.GetFormField("retrieval_password")
s.autoProcessHubRestore(w, r, customerID, password)
return
}
data := map[string]interface{}{
"CSRF": csrf,
"CustomerID": s.state.GetFormField("customer_id"),
}
s.render(w, "setup_hub_restore", data)
}
func (s *Server) handleRestore(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Redirect(w, r, "/setup", http.StatusFound)
return
}
if !validateCSRF(r) {
http.Error(w, "Invalid CSRF token", http.StatusForbidden)
return
}
source := r.FormValue("source")
switch source {
case "local":
drivePath := r.FormValue("drive_path")
historyFile := r.FormValue("history_file")
go s.executeLocalRestore(drivePath, historyFile)
case "hub":
go s.executeHubRestore()
default:
http.Error(w, "Invalid restore source", http.StatusBadRequest)
return
}
csrf := ensureCSRFToken(w, r)
data := map[string]interface{}{
"CSRF": csrf,
}
s.render(w, "setup_restore_exec", data)
}
func (s *Server) handleRestoreStatus(w http.ResponseWriter, r *http.Request) {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
resp := map[string]interface{}{
"running": s.restoreRunning,
"done": s.restoreDone,
"steps": s.restoreSteps,
"error": s.restoreError,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func (s *Server) handleFreshHub(w http.ResponseWriter, r *http.Request) {
csrf := ensureCSRFToken(w, r)
@@ -348,82 +214,6 @@ func (s *Server) handleLogo(w http.ResponseWriter, r *http.Request) {
// autoProcessHubRestore calls PullRecovery with pre-seeded credentials and
// renders the confirmation page directly, skipping the manual form.
// Falls back to the form with an error message on failure.
func (s *Server) autoProcessHubRestore(w http.ResponseWriter, r *http.Request, customerID, password string) {
hubURL := DefaultHubURL
s.logger.Printf("[INFO] Setup: auto-processing hub restore for %s (pre-seeded credentials)", customerID)
recovery, err := report.PullRecovery(hubURL, customerID, password)
if err != nil {
s.logger.Printf("[WARN] Setup: auto hub restore failed: %v — falling back to form", err)
var msg string
switch {
case isError(err, report.ErrHubUnreachable):
msg = "A Hub (hub.felhom.eu) nem elérhető. Ellenőrizze az internetkapcsolatot."
case isError(err, report.ErrAuthFailed):
msg = "Helytelen ügyfél-azonosító vagy jelszó."
case isError(err, report.ErrNotFound):
msg = "Ez az ügyfél-azonosító nem található a Hub-on."
default:
msg = fmt.Sprintf("Hiba történt: %v", err)
}
// Clear pre-seeded password so form is shown on next attempt
s.state.SetFormField("retrieval_password", "")
s.state.Save()
s.renderError(w, r, "setup_hub_restore", msg, customerID)
return
}
if s.isDebug() {
s.logger.Printf("[DEBUG] Setup: hub recovery received — hasInfra=%v, configLen=%d, versions=%d", recovery.HasInfraBackup, len(recovery.ConfigYAML), len(recovery.BackupVersions))
}
// If multiple versions available, show picker instead of auto-restoring
if len(recovery.BackupVersions) > 1 && recovery.HasInfraBackup {
s.logger.Printf("[INFO] Setup: %d backup versions available — showing version picker", len(recovery.BackupVersions))
// Store config for later use after version selection
s.state.SetFormField("hub_config_yaml", recovery.ConfigYAML)
s.state.Save()
csrf := ensureCSRFToken(w, r)
data := map[string]interface{}{
"CSRF": csrf,
"Versions": recovery.BackupVersions,
}
s.render(w, "setup_hub_versions", data)
return
}
// Single version or no versions — proceed directly
s.storeRecoveryAndRestore(w, r, recovery, customerID)
}
// storeRecoveryAndRestore stores recovery data in state and starts the restore goroutine.
func (s *Server) storeRecoveryAndRestore(w http.ResponseWriter, r *http.Request, recovery *report.RecoveryResponse, customerID string) {
s.state.SelectedBackup = &SelectedBackup{
Source: "hub",
CustomerID: customerID,
}
s.state.SetFormField("hub_config_yaml", recovery.ConfigYAML)
if recovery.HasInfraBackup && recovery.InfraBackup != nil {
ibJSON, _ := json.Marshal(recovery.InfraBackup)
s.state.SetFormField("hub_infra_backup", string(ibJSON))
s.state.SelectedBackup.Timestamp = recovery.InfraBackup.Timestamp
}
s.state.SetStep("restore-exec")
s.state.Save()
s.logger.Printf("[INFO] Setup: hub recovery stored (hasInfra=%v) — starting restore", recovery.HasInfraBackup)
go s.executeHubRestore()
csrf := ensureCSRFToken(w, r)
data := map[string]interface{}{
"CSRF": csrf,
}
s.render(w, "setup_restore_exec", data)
}
// autoProcessFreshHub calls PullConfig with pre-seeded credentials and
// proceeds with fresh install, skipping the manual form.
func (s *Server) autoProcessFreshHub(w http.ResponseWriter, r *http.Request, customerID, password string) {
@@ -466,104 +256,6 @@ func (s *Server) autoProcessFreshHub(w http.ResponseWriter, r *http.Request, cus
// --- Processing Logic ---
func (s *Server) processHubRestore(w http.ResponseWriter, r *http.Request) {
customerID := strings.TrimSpace(r.FormValue("customer_id"))
password := r.FormValue("password")
hubURL := DefaultHubURL
s.state.SetFormField("customer_id", customerID)
if customerID == "" || password == "" {
s.renderError(w, r, "setup_hub_restore", "Kérem töltse ki mindkét mezőt.", customerID)
return
}
if s.isDebug() {
s.logger.Printf("[DEBUG] Setup: hub restore — pulling recovery from %s for customer %s", hubURL, customerID)
}
recovery, err := report.PullRecovery(hubURL, customerID, password)
if err != nil {
var msg string
switch {
case isError(err, report.ErrHubUnreachable):
msg = "A Hub (hub.felhom.eu) nem elérhető. Ellenőrizze az internetkapcsolatot."
case isError(err, report.ErrAuthFailed):
msg = "Helytelen ügyfél-azonosító vagy jelszó."
case isError(err, report.ErrNotFound):
msg = "Ez az ügyfél-azonosító nem található a Hub-on."
default:
msg = fmt.Sprintf("Hiba történt: %v", err)
}
s.renderError(w, r, "setup_hub_restore", msg, customerID)
return
}
if s.isDebug() {
s.logger.Printf("[DEBUG] Setup: hub recovery received — hasInfra=%v, configLen=%d, versions=%d", recovery.HasInfraBackup, len(recovery.ConfigYAML), len(recovery.BackupVersions))
}
s.state.SetFormField("retrieval_password", password)
// If multiple versions available, show picker
if len(recovery.BackupVersions) > 1 && recovery.HasInfraBackup {
s.logger.Printf("[INFO] Setup: %d backup versions available — showing version picker", len(recovery.BackupVersions))
s.state.SetFormField("hub_config_yaml", recovery.ConfigYAML)
s.state.Save()
csrf := ensureCSRFToken(w, r)
data := map[string]interface{}{
"CSRF": csrf,
"Versions": recovery.BackupVersions,
}
s.render(w, "setup_hub_versions", data)
return
}
s.storeRecoveryAndRestore(w, r, recovery, customerID)
}
// handleHubVersionSelect processes the user's version selection from the Hub version picker.
func (s *Server) handleHubVersionSelect(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Redirect(w, r, "/setup/hub-restore", http.StatusFound)
return
}
if !validateCSRF(r) {
http.Error(w, "Invalid CSRF token", http.StatusForbidden)
return
}
versionStr := r.FormValue("version_id")
customerID := s.state.GetFormField("customer_id")
password := s.state.GetFormField("retrieval_password")
hubURL := DefaultHubURL
if customerID == "" || password == "" {
http.Redirect(w, r, "/setup/hub-restore", http.StatusFound)
return
}
var versionID int64
fmt.Sscanf(versionStr, "%d", &versionID)
s.logger.Printf("[INFO] Setup: user selected backup version %d for %s", versionID, customerID)
// Fetch the specific version
recovery, err := report.PullRecoveryVersion(hubURL, customerID, password, versionID)
if err != nil {
s.logger.Printf("[ERROR] Setup: failed to fetch version %d: %v", versionID, err)
csrf := ensureCSRFToken(w, r)
data := map[string]interface{}{
"CSRF": csrf,
"Error": fmt.Sprintf("Hiba a verzió letöltésekor: %v", err),
}
s.render(w, "setup_hub_versions", data)
return
}
s.storeRecoveryAndRestore(w, r, recovery, customerID)
}
func (s *Server) processFreshHub(w http.ResponseWriter, r *http.Request) {
customerID := strings.TrimSpace(r.FormValue("customer_id"))
password := r.FormValue("password")
@@ -676,232 +368,6 @@ func (s *Server) processManual(w http.ResponseWriter, r *http.Request) {
// --- Restore Execution ---
func (s *Server) executeLocalRestore(drivePath, historyFile string) {
s.restoreMu.Lock()
s.restoreRunning = true
s.restoreDone = false
s.restoreError = ""
s.restoreSteps = []RestoreStep{
{Label: "Mentés beolvasása...", Status: "running"},
{Label: "Konfiguráció visszaállítása...", Status: "pending"},
{Label: "Meghajtók csatolása...", Status: "pending"},
{Label: "Beállítás befejezése...", Status: "pending"},
}
s.restoreMu.Unlock()
// Step 1: Read backup (current or historical version)
var backupData []byte
var err error
if historyFile != "" {
backupData, _, err = backup.ReadLocalInfraBackupFromHistory(drivePath, historyFile)
} else {
backupData, _, err = backup.ReadLocalInfraBackup(drivePath)
}
if err != nil {
s.setRestoreError(0, fmt.Sprintf("Mentés olvasási hiba: %v", err))
return
}
var ib report.InfraBackup
if err := json.Unmarshal(backupData, &ib); err != nil {
s.setRestoreError(0, fmt.Sprintf("Mentés formátum hiba: %v", err))
return
}
s.setRestoreStepDone(0)
// Step 2: Write config files
s.setRestoreStepRunning(1)
if err := s.writeRestoredConfig(&ib); err != nil {
s.setRestoreError(1, fmt.Sprintf("Konfiguráció írási hiba: %v", err))
return
}
s.setRestoreStepDone(1)
// Step 3: Mount drives from disk layout
s.setRestoreStepRunning(2)
s.mountDrivesFromBackup(&ib)
s.setRestoreStepDone(2)
// Step 4: Finalize
s.setRestoreStepRunning(3)
// Save retrieval password from state if available
retrievalPw := s.state.GetFormField("retrieval_password")
if retrievalPw != "" {
sett, err := settings.Load(filepath.Join(s.dataDir, "settings.json"), s.logger)
if err == nil {
sett.SetRetrievalPassword(retrievalPw)
}
}
// Queue DR event
s.queueDREvent("local", ib.Timestamp, len(ib.DeployedStacks))
s.setRestoreStepDone(3)
s.restoreMu.Lock()
s.restoreRunning = false
s.restoreDone = true
s.restoreMu.Unlock()
s.logger.Printf("[INFO] Setup: local restore completed from %s", drivePath)
// Wait a moment for the UI to poll, then exit
time.Sleep(2 * time.Second)
s.finishSetup()
}
func (s *Server) executeHubRestore() {
s.restoreMu.Lock()
s.restoreRunning = true
s.restoreDone = false
s.restoreError = ""
s.restoreSteps = []RestoreStep{
{Label: "Konfiguráció visszaállítása...", Status: "running"},
{Label: "Meghajtók csatolása...", Status: "pending"},
{Label: "Beállítás befejezése...", Status: "pending"},
}
s.restoreMu.Unlock()
// Get stored data from state
configYAML := s.state.GetFormField("hub_config_yaml")
ibJSON := s.state.GetFormField("hub_infra_backup")
// Write controller.yaml
configPath := "/opt/docker/felhom-controller/controller.yaml"
if err := atomicWriteFile(configPath, []byte(configYAML), 0600); err != nil {
s.setRestoreError(0, fmt.Sprintf("Konfiguráció írási hiba: %v", err))
return
}
// Restore settings from infra backup if available
var restoredIB *report.InfraBackup
if ibJSON != "" {
var ib report.InfraBackup
if err := json.Unmarshal([]byte(ibJSON), &ib); err == nil {
s.restoreFromInfraBackup(&ib)
restoredIB = &ib
}
}
s.setRestoreStepDone(0)
// Step 2: Mount drives from disk layout
s.setRestoreStepRunning(1)
if restoredIB != nil {
s.mountDrivesFromBackup(restoredIB)
}
s.setRestoreStepDone(1)
// Step 3: Finalize
s.setRestoreStepRunning(2)
// Save retrieval password
retrievalPw := s.state.GetFormField("retrieval_password")
if retrievalPw != "" {
sett, err := settings.Load(filepath.Join(s.dataDir, "settings.json"), s.logger)
if err == nil {
sett.SetRetrievalPassword(retrievalPw)
}
}
// Queue DR event
stackCount := 0
timestamp := ""
if restoredIB != nil {
stackCount = len(restoredIB.DeployedStacks)
timestamp = restoredIB.Timestamp
}
s.queueDREvent("hub", timestamp, stackCount)
s.setRestoreStepDone(2)
s.restoreMu.Lock()
s.restoreRunning = false
s.restoreDone = true
s.restoreMu.Unlock()
s.logger.Printf("[INFO] Setup: Hub restore completed")
time.Sleep(2 * time.Second)
s.finishSetup()
}
// --- Config Writing ---
func (s *Server) writeRestoredConfig(ib *report.InfraBackup) error {
// Decode and write controller.yaml
if ib.ControllerConfigB64 != "" {
configData, err := base64.StdEncoding.DecodeString(ib.ControllerConfigB64)
if err != nil {
return fmt.Errorf("decoding controller.yaml: %w", err)
}
configPath := "/opt/docker/felhom-controller/controller.yaml"
if err := atomicWriteFile(configPath, configData, 0600); err != nil {
return fmt.Errorf("writing controller.yaml: %w", err)
}
}
s.restoreFromInfraBackup(ib)
return nil
}
func (s *Server) restoreFromInfraBackup(ib *report.InfraBackup) {
// Decode and write settings.json
if ib.SettingsJSONB64 != "" {
if data, err := base64.StdEncoding.DecodeString(ib.SettingsJSONB64); err == nil {
settingsPath := filepath.Join(s.dataDir, "settings.json")
if err := atomicWriteFile(settingsPath, data, 0644); err != nil {
s.logger.Printf("[WARN] Setup: failed to restore settings.json: %v", err)
}
}
}
// Restore restic password
if ib.ResticPassword != "" {
if data, err := base64.StdEncoding.DecodeString(ib.ResticPassword); err == nil {
pwFile := "/opt/docker/felhom-controller/data/restic-password"
if err := atomicWriteFile(pwFile, data, 0600); err != nil {
s.logger.Printf("[WARN] Setup: failed to restore restic password: %v", err)
}
}
}
// Restore encryption key for app.yaml secrets
if ib.EncryptionKeyB64 != "" {
if data, err := base64.StdEncoding.DecodeString(ib.EncryptionKeyB64); err == nil {
keyFile := filepath.Join(s.dataDir, "encryption.key")
if err := atomicWriteFile(keyFile, data, 0600); err != nil {
s.logger.Printf("[WARN] Setup: failed to restore encryption key: %v", err)
}
}
}
// Signal that FileBrowser's database should be reset on next startup.
// After restore, the DB has stale source preferences from the initial install.
flagPath := filepath.Join(s.dataDir, ".fb-reset")
_ = os.WriteFile(flagPath, []byte("restore"), 0644)
}
// mountDrivesFromBackup mounts drives from the infra backup's disk layout.
// Best-effort: logs warnings on failure but does not block restore.
func (s *Server) mountDrivesFromBackup(ib *report.InfraBackup) {
if len(ib.DiskLayout.Mounts) == 0 {
s.logger.Printf("[INFO] Setup: no drives in disk layout to mount")
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mounted, err := backup.MountDrivesFromLayout(ctx, ib.DiskLayout, s.logger)
if err != nil {
s.logger.Printf("[WARN] Setup: drive mounting error: %v", err)
}
if len(mounted) > 0 {
s.logger.Printf("[INFO] Setup: mounted %d drive(s): %v", len(mounted), mounted)
}
}
func (s *Server) writeFreshConfig(configYAML, retrievalPassword string) error {
configPath := "/opt/docker/felhom-controller/controller.yaml"
if err := atomicWriteFile(configPath, []byte(configYAML), 0600); err != nil {
@@ -1055,56 +521,6 @@ func (s *Server) finishSetup() {
os.Exit(0) // Docker restart policy will restart us
}
func (s *Server) queueDREvent(source, backupTimestamp string, stackCount int) {
sett, err := settings.Load(filepath.Join(s.dataDir, "settings.json"), s.logger)
if err != nil {
s.logger.Printf("[WARN] Setup: failed to load settings for DR event: %v", err)
return
}
details, _ := json.Marshal(map[string]interface{}{
"source": source,
"backup_timestamp": backupTimestamp,
"stacks_count": stackCount,
"controller_version": s.version,
})
sett.AddPendingEvent(settings.PendingEvent{
EventType: "disaster_recovery_completed",
Severity: "warning",
Message: "System restored from backup",
Details: string(details),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Server) setRestoreStepDone(idx int) {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
if idx < len(s.restoreSteps) {
s.restoreSteps[idx].Status = "done"
}
}
func (s *Server) setRestoreStepRunning(idx int) {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
if idx < len(s.restoreSteps) {
s.restoreSteps[idx].Status = "running"
}
}
func (s *Server) setRestoreError(idx int, msg string) {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
if idx < len(s.restoreSteps) {
s.restoreSteps[idx].Status = "failed"
s.restoreSteps[idx].Error = msg
}
s.restoreRunning = false
s.restoreError = msg
}
func (s *Server) render(w http.ResponseWriter, name string, data interface{}) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := s.tmpl.ExecuteTemplate(w, name, data); err != nil {
controller/internal/setup/scanner.go
-317
View File
@@ -1,317 +0,0 @@
package setup
import (
"bufio"
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
)
// DriveBackup represents a found infra backup on a drive.
type DriveBackup struct {
Device string `json:"device"`
Label string `json:"label"`
MountPoint string `json:"mount_point"`
CustomerID string `json:"customer_id"`
Timestamp string `json:"timestamp"`
CtrlVersion string `json:"controller_version"`
IntegrityOK bool `json:"integrity_ok"`
Error string `json:"error,omitempty"`
StackCount int `json:"stack_count"`
StackNames []string `json:"stack_names,omitempty"`
DiskCount int `json:"disk_count"`
IsHistory bool `json:"is_history"`
HistoryFile string `json:"history_file,omitempty"`
WasTempMounted bool `json:"-"`
}
// lsblkOutput represents the JSON output of lsblk.
type lsblkOutput struct {
Blockdevices []lsblkDevice `json:"blockdevices"`
}
type lsblkDevice struct {
Name string `json:"name"`
Path string `json:"path"`
FSType *string `json:"fstype"`
MountPoint *string `json:"mountpoint"`
Label *string `json:"label"`
Size interface{} `json:"size"` // string or int
Type string `json:"type"` // "disk", "part"
Children []lsblkDevice `json:"children,omitempty"`
}
// ScanDrivesForInfraBackups scans all block devices for .felhom-infra-backup/ directories.
func ScanDrivesForInfraBackups(logger *log.Logger, debug bool) ([]DriveBackup, error) {
logger.Printf("[INFO] Setup: scanning drives for infra backups...")
// Read currently mounted filesystems
mountedFS := readMountedFilesystems()
// Get root device to skip
rootDevices := getRootDevices()
// Run lsblk
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, "lsblk", "-J", "-o", "NAME,PATH,FSTYPE,MOUNTPOINT,LABEL,SIZE,TYPE").Output()
if err != nil {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
var lsblk lsblkOutput
if err := json.Unmarshal(out, &lsblk); err != nil {
return nil, fmt.Errorf("parsing lsblk: %w", err)
}
if debug {
logger.Printf("[DEBUG] Setup scan: lsblk returned %d block devices", len(lsblk.Blockdevices))
}
var results []DriveBackup
// Flatten all partitions
var partitions []lsblkDevice
for _, disk := range lsblk.Blockdevices {
if disk.Type == "part" {
partitions = append(partitions, disk)
}
for _, child := range disk.Children {
if child.Type == "part" {
partitions = append(partitions, child)
}
}
}
if debug {
logger.Printf("[DEBUG] Setup scan: found %d partitions to check, %d root devices to skip", len(partitions), len(rootDevices))
}
for _, part := range partitions {
// Skip partitions without filesystem
if part.FSType == nil || *part.FSType == "" || *part.FSType == "swap" {
continue
}
// Skip LUKS encrypted partitions
if *part.FSType == "crypto_LUKS" {
logger.Printf("[DEBUG] Setup: skipping LUKS partition %s", part.Path)
continue
}
// Skip LVM
if part.Type == "lvm" {
logger.Printf("[DEBUG] Setup: skipping LVM volume %s", part.Path)
continue
}
// Skip root partitions
if isRootPartition(part.Path, rootDevices) {
continue
}
partResults := scanPartition(part, mountedFS, logger)
results = append(results, partResults...)
}
logger.Printf("[INFO] Setup: drive scan complete — found %d backup(s)", countValid(results))
return results, nil
}
// CleanupTempMounts unmounts any partitions that were temporarily mounted during scanning.
func CleanupTempMounts(results []DriveBackup, logger *log.Logger) {
for _, r := range results {
if r.WasTempMounted && r.MountPoint != "" {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
exec.CommandContext(ctx, "umount", r.MountPoint).Run()
cancel()
os.Remove(r.MountPoint)
logger.Printf("[DEBUG] Setup: unmounted temp mount %s", r.MountPoint)
}
}
}
func scanPartition(part lsblkDevice, mountedFS map[string]string, logger *log.Logger) []DriveBackup {
label := ""
if part.Label != nil {
label = *part.Label
}
// Check if already mounted
var mountPoint string
var tempMounted bool
if part.MountPoint != nil && *part.MountPoint != "" {
mountPoint = *part.MountPoint
} else if mp, ok := mountedFS[part.Path]; ok {
mountPoint = mp
} else {
// Try to mount temporarily
tmpDir := filepath.Join("/mnt", ".felhom-scan", part.Name)
if err := os.MkdirAll(tmpDir, 0700); err != nil {
logger.Printf("[DEBUG] Setup: skip %s — cannot create temp dir: %v", part.Path, err)
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Try read-only mount
err := exec.CommandContext(ctx, "mount", "-o", "ro", part.Path, tmpDir).Run()
if err != nil {
// Retry with noload for journal errors
err = exec.CommandContext(ctx, "mount", "-o", "ro,noload", part.Path, tmpDir).Run()
}
if err != nil {
os.Remove(tmpDir)
logger.Printf("[DEBUG] Setup: skip %s — mount failed: %v", part.Path, err)
return nil
}
mountPoint = tmpDir
tempMounted = true
}
// Check for .felhom-infra-backup/
infraDir := backup.InfraBackupDir(mountPoint)
if _, err := os.Stat(infraDir); os.IsNotExist(err) {
if tempMounted {
exec.Command("umount", mountPoint).Run()
os.Remove(mountPoint)
}
return nil
}
var results []DriveBackup
// Read current backup
backupData, meta, err := backup.ReadLocalInfraBackup(mountPoint)
current := DriveBackup{
Device: part.Path,
Label: label,
MountPoint: mountPoint,
WasTempMounted: tempMounted,
}
if err != nil {
current.IntegrityOK = false
current.Error = err.Error()
if meta != nil {
current.CustomerID = meta.CustomerID
current.Timestamp = meta.Timestamp
current.CtrlVersion = meta.ControllerVersion
}
} else {
current.IntegrityOK = true
current.CustomerID = meta.CustomerID
current.Timestamp = meta.Timestamp
current.CtrlVersion = meta.ControllerVersion
backup.ParseBackupCounts(backupData, &current.StackCount, &current.StackNames, &current.DiskCount)
}
results = append(results, current)
logger.Printf("[INFO] Setup: found infra backup on %s (%s) — customer=%s, integrity=%v",
part.Path, label, current.CustomerID, current.IntegrityOK)
// Also scan history directory for older versions
history := backup.ReadLocalInfraHistory(mountPoint)
for _, hv := range history {
hResult := DriveBackup{
Device: part.Path,
Label: label,
MountPoint: mountPoint,
CustomerID: hv.CustomerID,
Timestamp: hv.Timestamp,
CtrlVersion: hv.ControllerVersion,
IntegrityOK: hv.IntegrityOK,
Error: hv.Error,
StackCount: hv.StackCount,
StackNames: hv.StackNames,
DiskCount: hv.DiskCount,
IsHistory: true,
HistoryFile: hv.HistoryFile,
}
results = append(results, hResult)
}
if len(history) > 0 {
logger.Printf("[INFO] Setup: found %d historical backup version(s) on %s", len(history), part.Path)
}
return results
}
func readMountedFilesystems() map[string]string {
result := make(map[string]string)
f, err := os.Open("/proc/mounts")
if err != nil {
return result
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) >= 2 {
result[fields[0]] = fields[1]
}
}
return result
}
func getRootDevices() map[string]bool {
result := make(map[string]bool)
mountedFS := readMountedFilesystems()
for dev, mp := range mountedFS {
if mp == "/" || mp == "/boot" || mp == "/boot/efi" {
result[dev] = true
}
}
return result
}
func isRootPartition(devPath string, rootDevices map[string]bool) bool {
return rootDevices[devPath]
}
func countValid(results []DriveBackup) int {
n := 0
for _, r := range results {
if r.IntegrityOK {
n++
}
}
return n
}
// runDriveScan runs the scan asynchronously and stores results on the Server.
func (s *Server) runDriveScan() {
results, err := ScanDrivesForInfraBackups(s.logger, s.isDebug())
// Clean up any temporary mounts created during scan
if results != nil {
CleanupTempMounts(results, s.logger)
}
s.scanMu.Lock()
defer s.scanMu.Unlock()
s.scanRunning = false
s.scanDone = true
if err != nil {
s.scanError = err.Error()
} else {
s.scanResults = results
}
}
controller/internal/storage/attach.go
-26
View File
@@ -1,26 +0,0 @@
package storage
import "log"
// AttachRequest holds parameters for attaching an existing partition
// without formatting — using a bind mount from a subfolder.
type AttachRequest struct {
DevicePath string // "/dev/sdb1" — must have an existing filesystem
MountName string // "hdd_1" → bind-mounts at /mnt/hdd_1
SubPath string // full path on raw mount to bind-mount (e.g., "/mnt/.felhom-raw/hdd_1/felhom_data")
Label string // Display label for the UI
SetDefault bool // Register as default storage path
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// DirEntry represents a directory entry returned by ListDirectories.
type DirEntry struct {
Name string `json:"name"`
Path string `json:"path"`
HasChildren bool `json:"has_children"`
}
// RawMountBase is the hidden staging directory where partitions are
// temporarily mounted for browsing before the final bind mount.
const RawMountBase = "/mnt/.felhom-raw"
controller/internal/storage/attach_linux.go
-482
View File
@@ -1,482 +0,0 @@
//go:build linux
package storage
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
// MountRaw temporarily mounts a partition at a staging path for browsing.
// The partition is mounted read-only so the user can inspect its contents
// before choosing a subfolder for the final bind mount.
// Returns the raw mount path (e.g., "/mnt/.felhom-raw/hdd_1").
func MountRaw(devicePath string) (string, error) {
// --- Validate device path ---
if !strings.HasPrefix(devicePath, "/dev/") {
return "", fmt.Errorf("érvénytelen eszközútvonal: /dev/-vel kell kezdődnie")
}
if strings.Contains(devicePath, "..") {
return "", fmt.Errorf("érvénytelen eszközútvonal: nem tartalmazhat ..-t")
}
if _, err := os.Stat(HostDevicePath(devicePath)); err != nil {
return "", fmt.Errorf("az eszköz nem létezik: %s", devicePath)
}
isSystem, err := IsSystemDisk(devicePath)
if err != nil {
return "", fmt.Errorf("rendszermeghajtó ellenőrzése sikertelen: %w", err)
}
if isSystem {
return "", fmt.Errorf("ez a rendszermeghajtó — nem csatolható")
}
mounted, err := IsDeviceMounted(devicePath)
if err != nil {
return "", fmt.Errorf("csatlakoztatási állapot ellenőrzése sikertelen: %w", err)
}
if mounted {
return "", fmt.Errorf("az eszköz már csatlakoztatva van")
}
// --- Detect filesystem ---
fsType, err := getBlkidValue(devicePath, "TYPE")
if err != nil || fsType == "" {
return "", fmt.Errorf("nincs fájlrendszer az eszközön (%s) — használja az inicializálás varázslót", devicePath)
}
// Get label for naming the raw mount directory
label, _ := getBlkidValue(devicePath, "LABEL")
uuid, _ := getBlkidValue(devicePath, "UUID")
// Choose a directory name: prefer label, fall back to UUID prefix
dirName := label
if dirName == "" && uuid != "" {
if len(uuid) > 8 {
dirName = uuid[:8]
} else {
dirName = uuid
}
}
if dirName == "" {
dirName = filepath.Base(devicePath) // "sdb1"
}
rawPath := filepath.Join(RawMountBase, dirName)
// Check if already raw-mounted (idempotent)
if inUse, _ := IsMountPathInUse(rawPath); inUse {
return rawPath, nil
}
// Create staging directory
if err := os.MkdirAll(rawPath, 0755); err != nil {
return "", fmt.Errorf("nem hozható létre a staging mappa: %w", err)
}
// Mount read-only for browsing
if out, err := exec.Command("mount", "-t", fsType, "-o", "defaults,noatime,ro",
HostDevicePath(devicePath), rawPath).CombinedOutput(); err != nil {
os.Remove(rawPath)
return "", fmt.Errorf("csatlakoztatás sikertelen: %s — %w", string(out), err)
}
return rawPath, nil
}
// ListDirectories returns the subdirectories at the given path.
// Only directories are returned; files, symlinks, and "lost+found" are excluded.
func ListDirectories(basePath string) ([]DirEntry, error) {
// Security: only allow browsing under the raw mount staging area
cleanPath := filepath.Clean(basePath)
if !strings.HasPrefix(cleanPath, RawMountBase) {
return nil, fmt.Errorf("érvénytelen útvonal: csak %s alatti mappák böngészhetők", RawMountBase)
}
entries, err := os.ReadDir(cleanPath)
if err != nil {
return nil, fmt.Errorf("mappa olvasása sikertelen: %w", err)
}
var dirs []DirEntry
for _, e := range entries {
if !e.IsDir() {
continue
}
name := e.Name()
// Skip lost+found and hidden directories
if name == "lost+found" || strings.HasPrefix(name, ".") {
continue
}
fullPath := filepath.Join(cleanPath, name)
// Check if this directory has subdirectories
hasChildren := false
if subEntries, err := os.ReadDir(fullPath); err == nil {
for _, se := range subEntries {
if se.IsDir() && se.Name() != "lost+found" && !strings.HasPrefix(se.Name(), ".") {
hasChildren = true
break
}
}
}
dirs = append(dirs, DirEntry{
Name: name,
Path: fullPath,
HasChildren: hasChildren,
})
}
return dirs, nil
}
// CreateDirectory creates a new directory at basePath/name.
// The raw mount is remounted read-write if needed.
func CreateDirectory(basePath, name string) (string, error) {
// Security: only allow creation under the raw mount staging area
cleanBase := filepath.Clean(basePath)
if !strings.HasPrefix(cleanBase, RawMountBase) {
return "", fmt.Errorf("érvénytelen útvonal: csak %s alatti mappák módosíthatók", RawMountBase)
}
// Validate directory name (same rules as mount names)
if err := ValidateMountName(name); err != nil {
return "", fmt.Errorf("érvénytelen mappanév: %w", err)
}
targetPath := filepath.Join(cleanBase, name)
// Check if already exists
if fi, err := os.Stat(targetPath); err == nil {
if fi.IsDir() {
return "", fmt.Errorf("a mappa már létezik: %s", name)
}
return "", fmt.Errorf("a cél már létezik és nem mappa")
}
// Remount read-write (the raw mount is initially read-only)
rawMountPoint := findRawMountPoint(cleanBase)
if rawMountPoint != "" {
_ = exec.Command("mount", "-o", "remount,rw", rawMountPoint).Run()
}
if err := os.MkdirAll(targetPath, 0755); err != nil {
return "", fmt.Errorf("mappa létrehozása sikertelen: %w", err)
}
_ = exec.Command("chown", "1000:1000", targetPath).Run()
return targetPath, nil
}
// FinalizeAttach creates the bind mount, fstab entries, and sets up permissions.
// Progress updates are sent on the progress channel.
// Returns the final mount path (/mnt/<name>) on success.
func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string, error) {
send := func(step, msg string, pct int) {
progress <- FormatProgress{Step: step, Message: msg, Percent: pct}
}
fail := func(step, msg string, err error) error {
errStr := ""
if err != nil {
errStr = err.Error()
}
progress <- FormatProgress{Step: "error", Message: msg, Error: errStr, Percent: 0}
if req.Logger != nil {
req.Logger.Printf("[ERROR] [storage] Failed to attach disk: %v", err)
}
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] [storage] FinalizeAttach: "+format, args...)
}
}
mountPath := "/mnt/" + req.MountName
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Attaching disk %s at %s", req.DevicePath, mountPath)
}
dbg("starting: device=%s mountName=%s subPath=%s", req.DevicePath, req.MountName, req.SubPath)
// --- Step 1: Validate ---
send("validating", "Paraméterek ellenőrzése...", 5)
if err := ValidateMountName(req.MountName); err != nil {
return "", fail("validating", "Érvénytelen csatlakoztatási név", err)
}
if !strings.HasPrefix(req.DevicePath, "/dev/") {
return "", fail("validating", "Érvénytelen eszközútvonal", fmt.Errorf("must start with /dev/"))
}
if strings.Contains(req.DevicePath, "..") {
return "", fail("validating", "Érvénytelen eszközútvonal", fmt.Errorf("must not contain .."))
}
// Validate subpath is under the raw mount area
cleanSub := filepath.Clean(req.SubPath)
if !strings.HasPrefix(cleanSub, RawMountBase) {
return "", fail("validating", "Érvénytelen almappa útvonal", fmt.Errorf("subpath must be under %s", RawMountBase))
}
if _, err := os.Stat(cleanSub); err != nil {
return "", fail("validating", "Az almappa nem létezik: "+cleanSub, err)
}
inUse, err := IsMountPathInUse(mountPath)
if err != nil {
return "", fail("validating", "Csatlakoztatási útvonal ellenőrzése sikertelen", err)
}
if inUse {
return "", fail("validating", "A csatlakoztatási útvonal már használatban van: "+mountPath, fmt.Errorf("mount path in use"))
}
send("validating", "Ellenőrzés kész", 15)
// --- Step 2: Ensure raw mount is read-write ---
send("mounting", "Fájlrendszer előkészítése...", 20)
rawMountPoint := findRawMountPoint(cleanSub)
if rawMountPoint != "" {
_ = exec.Command("mount", "-o", "remount,rw", rawMountPoint).Run()
}
// --- Step 3: Get device info for fstab ---
fsType, _ := getBlkidValue(req.DevicePath, "TYPE")
if fsType == "" {
fsType = "ext4" // fallback
}
uuid, err := getBlkidValue(req.DevicePath, "UUID")
if err != nil || uuid == "" {
return "", fail("mounting", "UUID lekérése sikertelen", fmt.Errorf("empty UUID for %s", req.DevicePath))
}
// Determine the raw mount directory name (the direct child of RawMountBase)
relFromBase := strings.TrimPrefix(cleanSub, RawMountBase+"/")
rawDirName := strings.SplitN(relFromBase, "/", 2)[0]
rawMountPath := filepath.Join(RawMountBase, rawDirName)
// Determine the subfolder relative to the raw mount
subRel := strings.TrimPrefix(cleanSub, rawMountPath)
subRel = strings.TrimPrefix(subRel, "/")
dbg("raw mount path: %s, sub relative: %q", rawMountPath, subRel)
send("mounting", "fstab bejegyzések hozzáadása...", 35)
// Backup fstab (non-fatal)
_ = BackupFstab(FstabPath)
// Fstab entry 1: raw partition mount
// Use nofail so a missing disk doesn't block boot
dbg("fstab entry 1: UUID=%s → %s (fstype=%s)", uuid, rawMountPath, fsType)
if err := AppendFstabEntry(FstabPath, uuid, rawMountPath, fsType, "defaults,nofail,noatime"); err != nil {
dbg("fstab raw mount entry failed: %v", err)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen (raw mount)", err)
}
// Fstab entry 2: bind mount from subfolder to final path
bindSource := cleanSub
dbg("fstab entry 2: bind %s → %s", bindSource, mountPath)
if err := appendBindFstabEntry(FstabPath, bindSource, mountPath); err != nil {
dbg("fstab bind entry failed: %v", err)
// Roll back the raw mount fstab entry
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen (bind mount)", err)
}
// --- Step 4: Create bind mount ---
send("mounting", fmt.Sprintf("Bind mount: %s → %s ...", cleanSub, mountPath), 50)
if err := os.MkdirAll(mountPath, 0755); err != nil {
_ = RemoveFstabEntry(FstabPath, uuid)
_ = removeBindFstabEntry(FstabPath, mountPath)
return "", fail("mounting", "Csatlakoztatási mappa nem hozható létre: "+mountPath, err)
}
dbg("bind mount: mount --bind %s %s", cleanSub, mountPath)
if out, err := exec.Command("mount", "--bind", cleanSub, mountPath).CombinedOutput(); err != nil {
dbg("bind mount failed: %s", string(out))
_ = RemoveFstabEntry(FstabPath, uuid)
_ = removeBindFstabEntry(FstabPath, mountPath)
return "", fail("mounting", "Bind mount sikertelen: "+string(out), err)
}
// Verify bind mount
dbg("verifying bind mount with findmnt")
verifyOut, verifyErr := exec.Command("findmnt", "-n", "-o", "SOURCE", "--target", mountPath).Output()
if verifyErr != nil || strings.TrimSpace(string(verifyOut)) == "" {
dbg("bind mount verification failed: findmnt returned %q err=%v", string(verifyOut), verifyErr)
_ = exec.Command("umount", mountPath).Run()
_ = RemoveFstabEntry(FstabPath, uuid)
_ = removeBindFstabEntry(FstabPath, mountPath)
return "", fail("mounting", "A bind mount nem ellenőrizhető", fmt.Errorf("mount point %s not found after bind mount", mountPath))
}
dbg("bind mount verified: source=%q", strings.TrimSpace(string(verifyOut)))
send("mounting", "Csatlakoztatva: "+mountPath, 70)
// --- Step 5: Permissions + subdirs ---
send("permissions", "Mappák létrehozása és jogosultságok beállítása...", 80)
_ = exec.Command("chown", "1000:1000", mountPath).Run()
for _, subdir := range []string{"felhom-data", "Dokumentumok"} {
dir := filepath.Join(mountPath, subdir)
if err := os.MkdirAll(dir, 0755); err == nil {
_ = exec.Command("chown", "1000:1000", dir).Run()
}
}
dbg("attach completed successfully: %s", mountPath)
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Disk attached successfully")
}
send("done", "Meghajtó sikeresen csatolva: "+mountPath, 100)
return mountPath, nil
}
// CleanupRawMount unmounts a staging raw mount and removes its directory.
// Called when the user cancels the attach wizard.
func CleanupRawMount(rawPath string) error {
cleanPath := filepath.Clean(rawPath)
if !strings.HasPrefix(cleanPath, RawMountBase) {
return fmt.Errorf("érvénytelen útvonal: csak %s alatti csatlakozások távolíthatók el", RawMountBase)
}
// Unmount
_ = exec.Command("umount", cleanPath).Run()
// Remove empty directory
_ = os.Remove(cleanPath)
return nil
}
// CleanupStaleRawMounts finds and removes raw mounts that have no corresponding
// bind mount — i.e., leftovers from an interrupted attach wizard.
// A raw mount is considered "in use" if fstab has a bind entry sourcing from it.
func CleanupStaleRawMounts() {
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return
}
// Read fstab to check for bind mount entries
fstabData, _ := os.ReadFile(FstabPath)
fstabLines := strings.Split(string(fstabData), "\n")
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
mountPoint := fields[1]
if !strings.HasPrefix(mountPoint, RawMountBase+"/") {
continue
}
// Only consider direct children of RawMountBase (e.g., /mnt/.felhom-raw/hdd_1)
rel := strings.TrimPrefix(mountPoint, RawMountBase+"/")
if strings.Contains(rel, "/") {
continue
}
// Check if any fstab bind entry sources from this raw mount path
inUse := false
for _, fl := range fstabLines {
fl = strings.TrimSpace(fl)
if fl == "" || strings.HasPrefix(fl, "#") {
continue
}
if strings.Contains(fl, "bind") && strings.HasPrefix(fl, mountPoint) {
inUse = true
break
}
}
if !inUse {
_ = exec.Command("umount", mountPoint).Run()
_ = os.Remove(mountPoint)
}
}
}
// --- helpers ---
// getBlkidValue runs blkid to get a single value (TYPE, UUID, LABEL) for a device.
func getBlkidValue(devicePath, tag string) (string, error) {
out, err := exec.Command("blkid", "-o", "value", "-s", tag, HostDevicePath(devicePath)).Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(out)), nil
}
// findRawMountPoint finds the mount point for a path under RawMountBase.
// E.g., for "/mnt/.felhom-raw/hdd_1/some/sub" it returns "/mnt/.felhom-raw/hdd_1".
func findRawMountPoint(path string) string {
cleanPath := filepath.Clean(path)
if !strings.HasPrefix(cleanPath, RawMountBase+"/") {
return ""
}
rel := strings.TrimPrefix(cleanPath, RawMountBase+"/")
parts := strings.SplitN(rel, "/", 2)
if len(parts) == 0 || parts[0] == "" {
return ""
}
return filepath.Join(RawMountBase, parts[0])
}
// appendBindFstabEntry appends a bind mount fstab entry.
func appendBindFstabEntry(fstabPath, source, target string) error {
existing, err := os.ReadFile(fstabPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("cannot read fstab: %w", err)
}
entry := fmt.Sprintf("\n# Bind mount (auto-generated by felhom-controller)\n%s\t%s\tnone\tbind,nofail\t0 0\n", source, target)
newContent := append(existing, []byte(entry)...)
return safeWriteFile(fstabPath, newContent, 0644)
}
// removeBindFstabEntry removes the bind mount fstab entry for the given target mount path.
func removeBindFstabEntry(fstabPath, targetMountPath string) error {
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("cannot read fstab: %w", err)
}
lines := strings.Split(string(data), "\n")
var kept []string
for i := 0; i < len(lines); i++ {
line := lines[i]
// Remove both the comment line and the bind mount line
if strings.Contains(line, "Bind mount (auto-generated by felhom-controller)") {
// Check if the next line is the actual bind entry for this target
if i+1 < len(lines) && fstabMatchesTarget(lines[i+1], targetMountPath) {
i++ // skip the bind line too
continue
}
}
if fstabMatchesTarget(line, targetMountPath) && strings.Contains(line, "bind") {
continue
}
kept = append(kept, line)
}
return safeWriteFile(fstabPath, []byte(strings.Join(kept, "\n")), 0644)
}
// fstabMatchesTarget parses an fstab line and checks if the mount target (field 2) matches exactly.
func fstabMatchesTarget(line, target string) bool {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
return false
}
fields := strings.Fields(line)
if len(fields) < 2 {
return false
}
return fields[1] == target
}
controller/internal/storage/attach_other.go
-33
View File
@@ -1,33 +0,0 @@
//go:build !linux
package storage
import "fmt"
// MountRaw is not supported on non-Linux platforms.
func MountRaw(devicePath string) (string, error) {
return "", fmt.Errorf("storage attach is only supported on Linux")
}
// ListDirectories is not supported on non-Linux platforms.
func ListDirectories(basePath string) ([]DirEntry, error) {
return nil, fmt.Errorf("storage attach is only supported on Linux")
}
// CreateDirectory is not supported on non-Linux platforms.
func CreateDirectory(basePath, name string) (string, error) {
return "", fmt.Errorf("storage attach is only supported on Linux")
}
// FinalizeAttach is not supported on non-Linux platforms.
func FinalizeAttach(req AttachRequest, progress chan<- FormatProgress) (string, error) {
return "", fmt.Errorf("storage attach is only supported on Linux")
}
// CleanupRawMount is not supported on non-Linux platforms.
func CleanupRawMount(rawPath string) error {
return fmt.Errorf("storage attach is only supported on Linux")
}
// CleanupStaleRawMounts is a no-op on non-Linux platforms.
func CleanupStaleRawMounts() {}
controller/internal/storage/format.go
-53
View File
@@ -1,53 +0,0 @@
package storage
import (
"bufio"
"fmt"
"log"
"strings"
)
// FormatRequest holds parameters for formatting and mounting a disk.
type FormatRequest struct {
DevicePath string // "/dev/sdb" or "/dev/sdb1"
MountName string // "hdd_1" → mounts at /mnt/hdd_1
Label string // Display label for the UI
CreatePartition bool // If true, create a single partition first (wipes disk)
SetDefault bool // Register as default storage path
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// FormatProgress tracks the formatting/mounting progress.
type FormatProgress struct {
Step string // "validating","partitioning","formatting","mounting","permissions","done","error"
Message string // Human-readable status
Error string // Non-empty if Step == "error"
Percent int // 0–100
}
// parseRsyncProgress parses a single line of rsync --info=progress2 output.
// Returns (bytesCopied, percent, ok).
func parseRsyncProgress(line string) (int64, int, bool) {
// Format: " 45,678,901 49% 12.34MB/s 0:00:30"
scanner := bufio.NewScanner(strings.NewReader(line))
scanner.Split(bufio.ScanWords)
var tokens []string
for scanner.Scan() {
tokens = append(tokens, scanner.Text())
}
if len(tokens) < 2 {
return 0, 0, false
}
bytesStr := strings.ReplaceAll(tokens[0], ",", "")
var bytesCopied int64
if _, err := fmt.Sscanf(bytesStr, "%d", &bytesCopied); err != nil {
return 0, 0, false
}
pctStr := strings.TrimSuffix(tokens[1], "%")
var pct int
if _, err := fmt.Sscanf(pctStr, "%d", &pct); err != nil {
return 0, 0, false
}
return bytesCopied, pct, true
}
controller/internal/storage/format_linux.go
-265
View File
@@ -1,265 +0,0 @@
//go:build linux
package storage
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// FormatAndMount formats a disk/partition and mounts it.
// Progress updates are sent on the progress channel.
// Returns the final mount path on success.
func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string, error) {
send := func(step, msg string, pct int) {
progress <- FormatProgress{Step: step, Message: msg, Percent: pct}
}
fail := func(step, msg string, err error) error {
errStr := ""
if err != nil {
errStr = err.Error()
}
progress <- FormatProgress{Step: "error", Message: msg, Error: errStr, Percent: 0}
if req.Logger != nil {
req.Logger.Printf("[ERROR] [storage] Format failed: %v", err)
}
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] [storage] FormatAndMount: "+format, args...)
}
}
mountPath := "/mnt/" + req.MountName
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Formatting disk %s as ext4", req.DevicePath)
}
dbg("starting: device=%s mountName=%s createPartition=%v", req.DevicePath, req.MountName, req.CreatePartition)
// --- Step 1: Validate ---
send("validating", "Eszköz ellenőrzése...", 5)
if err := ValidateMountName(req.MountName); err != nil {
return "", fail("validating", "Érvénytelen csatlakoztatási név", err)
}
// C6: Validate DevicePath to prevent path traversal from user-supplied input.
if !strings.HasPrefix(req.DevicePath, "/dev/") {
return "", fail("validating", "Érvénytelen eszközútvonal: /dev/-vel kell kezdődnie", fmt.Errorf("invalid device path: must start with /dev/"))
}
if strings.Contains(req.DevicePath, "..") {
return "", fail("validating", "Érvénytelen eszközútvonal: nem tartalmazhat ..-t", fmt.Errorf("invalid device path: must not contain .."))
}
if _, err := os.Stat(HostDevicePath(req.DevicePath)); err != nil {
return "", fail("validating", "Az eszköz nem létezik: "+req.DevicePath, err)
}
if req.CreatePartition {
// Whole-disk operation: block if any partition on this disk is a system partition.
isSystem, err := IsSystemDisk(req.DevicePath)
if err != nil {
return "", fail("validating", "Rendszermeghajtó ellenőrzése sikertelen", err)
}
if isSystem {
return "", fail("validating", "Ez a rendszermeghajtó — nem formázható!", fmt.Errorf("device is system disk"))
}
} else {
// Partition-only operation: block only if THIS specific partition is a system partition.
// Allows formatting empty partitions on the system disk (e.g., dedicated data partition).
isSysPart, err := IsSystemPartition(req.DevicePath)
if err != nil {
return "", fail("validating", "Rendszerpartíció ellenőrzése sikertelen", err)
}
if isSysPart {
return "", fail("validating", "Ez rendszerpartíció — nem formázható!", fmt.Errorf("partition is a system partition"))
}
}
mounted, err := IsDeviceMounted(req.DevicePath)
if err != nil {
return "", fail("validating", "Csatlakoztatási állapot ellenőrzése sikertelen", err)
}
if mounted {
return "", fail("validating", "Az eszköz már csatlakoztatva van", fmt.Errorf("device already mounted"))
}
inUse, err := IsMountPathInUse(mountPath)
if err != nil {
return "", fail("validating", "Csatlakoztatási útvonal ellenőrzése sikertelen", err)
}
if inUse {
return "", fail("validating", "A csatlakoztatási útvonal már használatban van: "+mountPath, fmt.Errorf("mount path in use"))
}
send("validating", "Ellenőrzés kész", 10)
// --- Step 2: Partition (if requested) ---
partDev := req.DevicePath
if req.CreatePartition {
// Wipe existing partition table and filesystem signatures first
// H18: Log wipefs errors instead of silently discarding them.
dbg("step wipefs: wipefs -a %s", HostDevicePath(req.DevicePath))
send("partitioning", fmt.Sprintf("wipefs -a %s ...", HostDevicePath(req.DevicePath)), 12)
if err := exec.Command("wipefs", "-a", HostDevicePath(req.DevicePath)).Run(); err != nil {
// Non-fatal: some systems don't have wipefs; continue anyway
dbg("wipefs failed (non-fatal): %v", err)
send("partitioning", fmt.Sprintf("[WARN] wipefs sikertelen %s: %v (folytatás)", req.DevicePath, err), 13)
} else {
dbg("wipefs completed successfully")
}
time.Sleep(500 * time.Millisecond)
// Create GPT with single partition spanning whole disk.
// ",," = start=default, size=default(fill disk), type=default(Linux filesystem GUID).
// --force: overwrite even if device appears busy.
// --wipe always: wipe filesystem signatures from newly created partitions.
dbg("step sfdisk: sfdisk --force --wipe always %s", HostDevicePath(req.DevicePath))
send("partitioning", fmt.Sprintf("sfdisk --force --wipe always %s ...", HostDevicePath(req.DevicePath)), 15)
sfdiskInput := "label: gpt\n,,\n"
cmd := exec.Command("sfdisk", "--force", "--wipe", "always", HostDevicePath(req.DevicePath))
cmd.Stdin = strings.NewReader(sfdiskInput)
if out, err := cmd.CombinedOutput(); err != nil {
dbg("sfdisk failed: %s", util.TruncateStr(string(out), 500))
return "", fail("partitioning", "Partícionálás sikertelen: "+string(out), err)
} else {
dbg("sfdisk output: %s", util.TruncateStr(string(out), 500))
}
_ = exec.Command("partprobe", HostDevicePath(req.DevicePath)).Run()
time.Sleep(2 * time.Second)
partDev = req.DevicePath + "1"
if strings.Contains(req.DevicePath, "nvme") || strings.Contains(req.DevicePath, "mmcblk") {
partDev = req.DevicePath + "p1"
}
if _, err := os.Stat(HostDevicePath(partDev)); err != nil {
return "", fail("partitioning", "Partíció nem található a létrehozás után: "+partDev, err)
}
send("partitioning", "Partíció létrehozva: "+partDev, 25)
}
// --- Step 3: Format ---
// Use ASCII-safe mount name for ext4 filesystem label (16-byte limit).
// The display label (req.Label) stays in settings.json for the UI.
fsLabel := req.MountName
if len(fsLabel) > 16 {
fsLabel = fsLabel[:16]
}
dbg("step mkfs.ext4: mkfs.ext4 -L %s -F %s", fsLabel, HostDevicePath(partDev))
send("formatting", fmt.Sprintf("mkfs.ext4 -L %s -F %s ...", fsLabel, HostDevicePath(partDev)), 30)
mkfsCmd := exec.Command("mkfs.ext4", "-L", fsLabel, "-F", HostDevicePath(partDev))
var mkfsOut bytes.Buffer
mkfsCmd.Stdout = &mkfsOut
mkfsCmd.Stderr = &mkfsOut
if err := mkfsCmd.Run(); err != nil {
dbg("mkfs.ext4 failed: %s", util.TruncateStr(mkfsOut.String(), 500))
return "", fail("formatting", "Formázás sikertelen: "+mkfsOut.String(), err)
}
dbg("mkfs.ext4 output: %s", util.TruncateStr(mkfsOut.String(), 500))
send("formatting", "Formázás kész", 60)
// --- Step 4: Mount ---
if err := os.MkdirAll(mountPath, 0755); err != nil {
return "", fail("mounting", "Csatlakoztatási mappa nem hozható létre: "+mountPath, err)
}
dbg("step blkid: blkid -s UUID -o value %s", HostDevicePath(partDev))
send("mounting", fmt.Sprintf("UUID lekérése: blkid %s ...", HostDevicePath(partDev)), 65)
uuidOut, err := exec.Command("blkid", "-s", "UUID", "-o", "value", HostDevicePath(partDev)).Output()
if err != nil {
dbg("blkid UUID failed: %v", err)
return "", fail("mounting", "UUID lekérése sikertelen", err)
}
uuid := strings.TrimSpace(string(uuidOut))
dbg("blkid returned UUID=%q", uuid)
if uuid == "" {
return "", fail("mounting", "UUID üres a formázás után", fmt.Errorf("empty UUID"))
}
// Backup fstab (non-fatal)
_ = BackupFstab(FstabPath)
dbg("step fstab: appending UUID=%s mountPath=%s fstype=ext4", uuid, mountPath)
if err := AppendFstabEntry(FstabPath, uuid, mountPath, "ext4", "defaults,nofail,noatime"); err != nil {
dbg("fstab append failed: %v", err)
return "", fail("mounting", "fstab bejegyzés hozzáadása sikertelen", err)
}
dbg("fstab entry added successfully")
// Mount by device path explicitly — container's /etc/fstab != host fstab,
// so "mount /mnt/hdd_1" (fstab lookup) won't work.
dbg("step mount: mount -t ext4 -o defaults,noatime %s %s", HostDevicePath(partDev), mountPath)
send("mounting", fmt.Sprintf("mount -t ext4 %s %s ...", HostDevicePath(partDev), mountPath), 70)
if out, err := exec.Command("mount", "-t", "ext4", "-o", "defaults,noatime",
HostDevicePath(partDev), mountPath).CombinedOutput(); err != nil {
dbg("mount failed: %s", util.TruncateStr(string(out), 500))
// H19: Roll back fstab entry to prevent orphaned entry that hangs system on reboot.
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "Csatlakoztatás sikertelen: "+string(out), err)
}
// Verify mount actually worked (don't just trust exit code)
dbg("step verify: findmnt -n -o SOURCE --target %s", mountPath)
verifyOut, verifyErr := exec.Command("findmnt", "-n", "-o", "SOURCE", "--target", mountPath).Output()
if verifyErr != nil || strings.TrimSpace(string(verifyOut)) == "" {
dbg("mount verification failed: findmnt returned %q err=%v", string(verifyOut), verifyErr)
// H19: Also roll back fstab if mount verify fails.
_ = RemoveFstabEntry(FstabPath, uuid)
return "", fail("mounting", "A csatlakoztatás nem ellenőrizhető: mount sikerült, de a meghajtó nem látható",
fmt.Errorf("mount point %s not found after mount", mountPath))
}
dbg("mount verified: findmnt source=%q", strings.TrimSpace(string(verifyOut)))
send("mounting", "Csatlakoztatva: "+mountPath, 80)
// --- Step 5: Permissions + subdirs ---
send("permissions", "Mappák létrehozása és jogosultságok beállítása...", 85)
_ = exec.Command("chown", "1000:1000", mountPath).Run()
for _, subdir := range []string{"felhom-data", "Dokumentumok"} {
dir := filepath.Join(mountPath, subdir)
if err := os.MkdirAll(dir, 0755); err == nil {
_ = exec.Command("chown", "1000:1000", dir).Run()
}
}
dbg("format and mount completed successfully: %s", mountPath)
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Disk formatted and mounted at %s", mountPath)
}
send("done", "Meghajtó sikeresen inicializálva: "+mountPath, 100)
return mountPath, nil
}
// GetDeviceUUID returns the UUID of a block device/partition.
func GetDeviceUUID(devicePath string) (string, error) {
out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", HostDevicePath(devicePath)).Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(out)), nil
}
// ReadFstab reads the current fstab content.
func ReadFstab() (string, error) {
data, err := os.ReadFile(FstabPath)
if err != nil {
data, err = os.ReadFile("/etc/fstab")
if err != nil {
return "", err
}
}
return string(data), nil
}
controller/internal/storage/format_other.go
-20
View File
@@ -1,20 +0,0 @@
//go:build !linux
package storage
import "fmt"
// FormatAndMount is not supported on non-Linux platforms.
func FormatAndMount(req FormatRequest, progress chan<- FormatProgress) (string, error) {
return "", fmt.Errorf("storage init is only supported on Linux")
}
// GetDeviceUUID is not supported on non-Linux platforms.
func GetDeviceUUID(devicePath string) (string, error) {
return "", fmt.Errorf("storage init is only supported on Linux")
}
// ReadFstab is not supported on non-Linux platforms.
func ReadFstab() (string, error) {
return "", fmt.Errorf("storage init is only supported on Linux")
}
controller/internal/storage/migrate.go
-497
View File
@@ -1,497 +0,0 @@
package storage
import (
"bufio"
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// MigrateRequest holds parameters for migrating app data.
type MigrateRequest struct {
StackName string // e.g., "immich"
DisplayName string // e.g., "Immich"
CurrentHDDPath string // e.g., "/mnt/hdd_placeholder"
TargetPath string // e.g., "/mnt/hdd_1"
HDDMounts []string // host-side paths to rsync (e.g., ["/mnt/hdd_placeholder/storage/immich"])
Logger *log.Logger // Optional logger for debug output
Debug bool // Enable debug logging
}
// MigrateProgress tracks migration state.
type MigrateProgress struct {
Step string // "stopping","copying","updating","starting","done","error","rolling_back"
Message string
BytesCopied int64
BytesTotal int64
Percent int
Error string
ElapsedSeconds int
}
// StopFunc stops an app's containers. Returns error if stop fails.
type StopFunc func(stackName string) error
// StartFunc starts an app's containers. Returns error if start fails.
type StartFunc func(stackName string) error
// UpdateHDDPathFunc updates the HDD_PATH in app.yaml. Returns error on failure.
type UpdateHDDPathFunc func(stackName, newPath string) error
// MigrateAppData moves app data from current to target storage path.
// stopFn and startFn are called to stop/start the app containers.
// updateFn is called to update the app's HDD_PATH configuration.
func MigrateAppData(
req MigrateRequest,
stopFn StopFunc,
startFn StartFunc,
updateFn UpdateHDDPathFunc,
progress chan<- MigrateProgress,
) error {
start := time.Now()
send := func(step, msg string, pct int, bytesCopied, bytesTotal int64) {
progress <- MigrateProgress{
Step: step,
Message: msg,
Percent: pct,
BytesCopied: bytesCopied,
BytesTotal: bytesTotal,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
fail := func(step, msg string, err error) error {
errStr := ""
if err != nil {
errStr = err.Error()
}
progress <- MigrateProgress{
Step: "error",
Message: msg,
Error: errStr,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
if req.Logger != nil {
req.Logger.Printf("[ERROR] [storage] Migration %s failed at %s: %v", req.StackName, step, err)
}
return fmt.Errorf("%s: %w", msg, err)
}
dbg := func(format string, args ...interface{}) {
if req.Logger != nil && req.Debug {
req.Logger.Printf("[DEBUG] [storage] MigrateAppData: "+format, args...)
}
}
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Starting migration: stack=%s from=%s to=%s", req.StackName, req.CurrentHDDPath, req.TargetPath)
}
dbg("starting migration: stack=%s from=%s to=%s mounts=%d", req.StackName, req.CurrentHDDPath, req.TargetPath, len(req.HDDMounts))
// --- Step 1: Validate ---
if req.CurrentHDDPath == "" {
return fail("validating", "A jelenlegi tárhely nem megadott", fmt.Errorf("empty current HDD path"))
}
if req.TargetPath == "" {
return fail("validating", "A cél tárhely nem megadott", fmt.Errorf("empty target path"))
}
if req.CurrentHDDPath == req.TargetPath {
return fail("validating", "A forrás és a cél tárhely azonos", fmt.Errorf("source equals target"))
}
if _, err := os.Stat(req.TargetPath); err != nil {
return fail("validating", "A cél tárhely nem létezik: "+req.TargetPath, err)
}
if len(req.HDDMounts) == 0 {
return fail("validating", "Nincsenek HDD csatlakozások az alkalmazáshoz", fmt.Errorf("no HDD mounts"))
}
// Estimate total size
var totalBytes int64
for _, m := range req.HDDMounts {
if info, err := os.Stat(m); err == nil && info.IsDir() {
totalBytes += dirSize(m)
}
}
dbg("estimated total size: %s (%d bytes)", bytesHuman(totalBytes), totalBytes)
// Check free space on target
freeBytes := getFreeBytes(req.TargetPath)
dbg("target free space: %s (%d bytes)", bytesHuman(freeBytes), freeBytes)
if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes {
return fail("validating", fmt.Sprintf(
"Nincs elég szabad hely a céltárolón: szükséges ~%s, szabad %s",
bytesHuman(totalBytes), bytesHuman(freeBytes),
), fmt.Errorf("insufficient disk space"))
}
send("stopping", "Alkalmazás leállítása...", 5, 0, totalBytes)
// --- Step 2: Stop app ---
if err := stopFn(req.StackName); err != nil {
return fail("stopping", "Alkalmazás leállítása sikertelen", err)
}
send("stopping", "Alkalmazás leállítva", 10, 0, totalBytes)
// --- Step 3: rsync ---
dbg("starting rsync phase: %d mount(s) to copy", len(req.HDDMounts))
var bytesCopied int64
for i, srcPath := range req.HDDMounts {
// Determine destination path: replace CurrentHDDPath prefix with TargetPath.
// H13: Require trailing separator to prevent /mnt/hdd matching /mnt/hdd_backup/data.
if srcPath != req.CurrentHDDPath && !strings.HasPrefix(srcPath, req.CurrentHDDPath+"/") {
dbg("skipping mount %s (not under %s)", srcPath, req.CurrentHDDPath)
continue
}
relPath := strings.TrimPrefix(srcPath, req.CurrentHDDPath)
dstPath := filepath.Join(req.TargetPath, relPath)
dbg("rsync %d/%d: %s → %s", i+1, len(req.HDDMounts), srcPath, dstPath)
// Ensure destination parent exists
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
// Rollback
send("rolling_back", "Hiba: mappa létrehozása sikertelen, visszagörgetés...", 0, bytesCopied, totalBytes)
_ = startFn(req.StackName)
return fail("copying", "Cél mappa létrehozása sikertelen: "+filepath.Dir(dstPath), err)
}
mountPct := 10 + (i * 60 / len(req.HDDMounts))
send("copying", fmt.Sprintf("Adatok másolása (%d/%d): %s...", i+1, len(req.HDDMounts), filepath.Base(srcPath)),
mountPct, bytesCopied, totalBytes)
var rsyncErr error
bytesCopied, rsyncErr = runRsync(srcPath, dstPath, totalBytes, bytesCopied, mountPct, progress, start)
if rsyncErr != nil {
// Rollback
send("rolling_back", "rsync sikertelen, alkalmazás visszaállítása az eredeti tárolóra...", 0, bytesCopied, totalBytes)
_ = startFn(req.StackName)
return fail("copying", "Adatmásolás sikertelen", rsyncErr)
}
}
send("updating", "Konfiguráció frissítése...", 75, bytesCopied, totalBytes)
// --- Step 4: Update app.yaml HDD_PATH ---
dbg("updating config: HDD_PATH %s → %s for stack %s", req.CurrentHDDPath, req.TargetPath, req.StackName)
if err := updateFn(req.StackName, req.TargetPath); err != nil {
send("rolling_back", "Konfiguráció frissítése sikertelen, visszaállítás...", 0, bytesCopied, totalBytes)
_ = startFn(req.StackName)
return fail("updating", "HDD_PATH frissítése sikertelen", err)
}
send("starting", "Alkalmazás indítása az új tárolóról...", 85, bytesCopied, totalBytes)
// --- Step 5: Start app ---
if err := startFn(req.StackName); err != nil {
// Revert config and restart with old path
_ = updateFn(req.StackName, req.CurrentHDDPath)
_ = startFn(req.StackName)
return fail("starting", "Alkalmazás indítása sikertelen az új tárolóról", err)
}
if req.Logger != nil {
req.Logger.Printf("[INFO] [storage] Migration completed: stack=%s elapsed=%ds", req.StackName, int(time.Since(start).Seconds()))
}
dbg("migration completed: stack=%s bytesCopied=%d elapsed=%ds", req.StackName, bytesCopied, int(time.Since(start).Seconds()))
send("done",
fmt.Sprintf("Áthelyezés kész! Az alkalmazás az új tárolóról fut. (Régi adat: %s, idő: %ds)",
req.CurrentHDDPath, int(time.Since(start).Seconds())),
100, bytesCopied, totalBytes)
return nil
}
// runRsync runs rsync from srcPath to dstPath and reports progress.
func runRsync(srcPath, dstPath string, totalBytes, prevCopied int64, basePct int, progress chan<- MigrateProgress, start time.Time) (int64, error) {
// Ensure src ends with / for rsync to sync contents (not the directory itself)
if !strings.HasSuffix(srcPath, "/") {
srcPath += "/"
}
cmd := exec.Command(
"rsync", "-a", "--info=progress2", "--human-readable",
srcPath, dstPath,
)
stdout, err := cmd.StdoutPipe()
if err != nil {
return prevCopied, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return prevCopied, err
}
if err := cmd.Start(); err != nil {
return prevCopied, fmt.Errorf("rsync start failed: %w", err)
}
var bytesCopied int64 = prevCopied
var mu sync.Mutex
// Parse stdout progress
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
if b, pct, ok := parseRsyncProgress(line); ok {
mu.Lock()
bytesCopied = prevCopied + b
// Scale pct into our range
scaledPct := basePct + pct*40/100
if scaledPct > 99 {
scaledPct = 99
}
mu.Unlock()
progress <- MigrateProgress{
Step: "copying",
Message: fmt.Sprintf("Adatok másolása... %s / %s", bytesHuman(b), bytesHuman(totalBytes)),
Percent: scaledPct,
BytesCopied: bytesCopied,
BytesTotal: totalBytes,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
}
}()
var stderrBuf strings.Builder
io.Copy(&stderrBuf, stderr)
if err := cmd.Wait(); err != nil {
// H11: Read bytesCopied under lock to avoid data race with the progress goroutine.
mu.Lock()
copied := bytesCopied
mu.Unlock()
return copied, fmt.Errorf("rsync failed: %w — %s", err, stderrBuf.String())
}
mu.Lock()
finalCopied := bytesCopied
mu.Unlock()
return finalCopied, nil
}
// dirSize returns the total bytes in a directory (best-effort).
func dirSize(path string) int64 {
var total int64
filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return nil
}
total += info.Size()
return nil
})
return total
}
// getFreeBytes returns available bytes on the filesystem at path.
func getFreeBytes(path string) int64 {
// Use df to get available bytes — works cross-platform within Linux container
out, err := exec.Command("df", "-B1", "--output=avail", path).Output()
if err != nil {
return 0
}
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
if len(lines) < 2 {
return 0
}
var avail int64
fmt.Sscanf(strings.TrimSpace(lines[1]), "%d", &avail)
return avail
}
// bytesHuman converts a byte count to human-readable string.
func bytesHuman(b int64) string {
const (
KB = 1024
MB = KB * 1024
GB = MB * 1024
)
switch {
case b >= GB:
return fmt.Sprintf("%.1f GB", float64(b)/float64(GB))
case b >= MB:
return fmt.Sprintf("%.0f MB", float64(b)/float64(MB))
case b >= KB:
return fmt.Sprintf("%.0f KB", float64(b)/float64(KB))
default:
return fmt.Sprintf("%d B", b)
}
}
// BackupTrigger allows triggering backup operations without importing the backup package.
type BackupTrigger interface {
TryRunDriveBackup(ctx context.Context, drivePath string) error
}
// MigrateOptions holds optional configuration for enhanced migration.
type MigrateOptions struct {
AutoDeleteStale bool // delete old data from source after success (default true)
}
// MigrateOrchestrator wraps MigrateAppData with backup-aware pre/post steps.
type MigrateOrchestrator struct {
Sett *settings.Settings
BackupTrigger BackupTrigger // nil if backup disabled
Logger *log.Logger
}
// RunEnhancedMigration runs MigrateAppData with additional pre/post-migration steps:
// - Copy DB dumps from source to destination drive
// - Clear Tier 2 config if destination conflicts with cross-drive target
// - Optionally delete stale data from source drive
// - Trigger immediate Tier 1 backup on destination drive
func (o *MigrateOrchestrator) RunEnhancedMigration(
req MigrateRequest,
stopFn StopFunc,
startFn StartFunc,
updateFn UpdateHDDPathFunc,
opts MigrateOptions,
progress chan<- MigrateProgress,
) error {
start := time.Now()
// Pre-flight: detect Tier 2 conflict
var tier2WillClear bool
cfg := o.Sett.GetCrossDriveConfig(req.StackName)
if cfg != nil && cfg.Enabled && cfg.DestinationPath != "" {
// If destination is under the target drive, the Tier 2 backup would point
// to the same drive the app now lives on — no redundancy, so we clear it.
if strings.HasPrefix(cfg.DestinationPath, req.TargetPath) || cfg.DestinationPath == req.TargetPath {
tier2WillClear = true
o.Logger.Printf("[INFO] [storage] Migration %s: Tier 2 will be cleared (dest %s is under target %s)",
req.StackName, cfg.DestinationPath, req.TargetPath)
}
}
// Run core migration (stop, rsync, update config, start).
// Intercept the "done" step from MigrateAppData — we have post-steps to run.
innerCh := make(chan MigrateProgress, 64)
innerDone := make(chan struct{})
go func() {
for p := range innerCh {
if p.Step == "done" {
// Suppress MigrateAppData's "done" — we'll send our own after post-steps.
continue
}
progress <- p
}
close(innerDone)
}()
if err := MigrateAppData(req, stopFn, startFn, updateFn, innerCh); err != nil {
close(innerCh)
<-innerDone
return err
}
close(innerCh)
<-innerDone // wait for forwarding goroutine to finish
// --- Post-migration steps (all non-fatal) ---
// 1. Copy DB dumps from source to destination
srcDBDumps := appbackup.AppDBDumpPath(req.CurrentHDDPath, req.StackName)
dstDBDumps := appbackup.AppDBDumpPath(req.TargetPath, req.StackName)
if _, err := os.Stat(srcDBDumps); err == nil {
if err := os.MkdirAll(filepath.Dir(dstDBDumps), 0755); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: failed to create DB dump dir: %v", req.StackName, err)
} else {
cmd := exec.Command("rsync", "-a", srcDBDumps+"/", dstDBDumps+"/")
if out, err := cmd.CombinedOutput(); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: DB dump copy failed: %v — %s", req.StackName, err, string(out))
} else {
o.Logger.Printf("[INFO] [storage] Migration %s: DB dumps copied to %s", req.StackName, dstDBDumps)
}
}
}
// 2. Clear Tier 2 if conflict
if tier2WillClear {
if err := o.Sett.SetCrossDriveConfig(req.StackName, nil); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: failed to clear Tier 2 config: %v", req.StackName, err)
} else {
o.Logger.Printf("[INFO] [storage] Migration %s: Tier 2 cross-drive config cleared (dest was on same drive)", req.StackName)
}
}
// 3. Auto-delete stale data from source
if opts.AutoDeleteStale {
progress <- MigrateProgress{
Step: "cleaning",
Message: "Régi adatok törlése a forrás meghajtóról...",
Percent: 92,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
// Delete app data from source
for _, srcPath := range req.HDDMounts {
if !strings.HasPrefix(srcPath, req.CurrentHDDPath+"/") && srcPath != req.CurrentHDDPath {
continue
}
if err := os.RemoveAll(srcPath); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: failed to delete stale data %s: %v", req.StackName, srcPath, err)
} else {
o.Logger.Printf("[INFO] [storage] Migration %s: deleted stale data %s", req.StackName, srcPath)
}
}
// Delete DB dumps from source
if _, err := os.Stat(srcDBDumps); err == nil {
if err := os.RemoveAll(srcDBDumps); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: failed to delete stale DB dumps %s: %v", req.StackName, srcDBDumps, err)
} else {
o.Logger.Printf("[INFO] [storage] Migration %s: deleted stale DB dumps %s", req.StackName, srcDBDumps)
}
}
}
// 4. Trigger immediate Tier 1 backup on destination drive
if o.BackupTrigger != nil {
progress <- MigrateProgress{
Step: "backing_up",
Message: "Biztonsági mentés indítása az új meghajtón...",
Percent: 95,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
if err := o.BackupTrigger.TryRunDriveBackup(context.Background(), req.TargetPath); err != nil {
o.Logger.Printf("[WARN] [storage] Migration %s: post-migration backup failed: %v", req.StackName, err)
progress <- MigrateProgress{
Step: "backing_up",
Message: "Biztonsági mentés nem indítható (másik mentés fut)",
Percent: 96,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
} else {
o.Logger.Printf("[INFO] [storage] Migration %s: post-migration backup completed for %s", req.StackName, req.TargetPath)
}
}
// Final done step with enhanced info
msg := fmt.Sprintf("Áthelyezés kész! Az alkalmazás az új tárolóról fut. (idő: %ds)", int(time.Since(start).Seconds()))
if tier2WillClear {
msg += " A 2. szintű mentés törlésre került."
}
progress <- MigrateProgress{
Step: "done",
Message: msg,
Percent: 100,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
return nil
}
controller/internal/storage/migrate_drive.go
-539
View File
@@ -1,539 +0,0 @@
package storage
import (
"bufio"
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// StackProviderForMigration abstracts stack operations needed by drive migration.
type StackProviderForMigration interface {
ListDeployedStacks() []StackSummaryForMigration
GetStackHDDPath(name string) string
StopStack(name string) error
StartStack(name string) error
UpdateStackHDDPath(name, newPath string) error
StackExists(name string) bool
}
// StackSummaryForMigration holds minimal stack info for drive migration.
type StackSummaryForMigration struct {
Name string
DisplayName string
}
// DriveMigrateRequest holds parameters for migrating all apps from one drive to another.
type DriveMigrateRequest struct {
SourcePath string // e.g., "/mnt/hdd_1"
DestPath string // e.g., "/mnt/hdd_2"
}
// DriveMigrateProgress tracks drive migration state.
type DriveMigrateProgress struct {
Step string // "validating","stopping","copying","verifying","configuring","starting","backup","done","error","rolling_back"
Message string
BytesCopied int64
BytesTotal int64
Percent int
Error string
ElapsedSeconds int
Detail string // sub-step detail (e.g., which app is being configured)
}
// DriveMigrator orchestrates full drive migration.
type DriveMigrator struct {
Sett *settings.Settings
StackProvider StackProviderForMigration
BackupTrigger BackupTrigger
AlertRefresh func()
PushHubReport func()
PushInfraBackup func()
SyncFBMounts func()
Logger *log.Logger
mu sync.Mutex
active bool // global migration lock
}
// IsActive returns whether a full drive migration is currently in progress.
func (dm *DriveMigrator) IsActive() bool {
dm.mu.Lock()
defer dm.mu.Unlock()
return dm.active
}
// rollbackAction describes a reversible action in the migration transaction.
type rollbackAction struct {
description string
undo func() error
}
// migrationTx is a transaction log that enables reverse-order rollback.
type migrationTx struct {
actions []rollbackAction
logger *log.Logger
}
func (tx *migrationTx) add(desc string, undoFn func() error) {
tx.actions = append(tx.actions, rollbackAction{description: desc, undo: undoFn})
}
func (tx *migrationTx) rollback() {
for i := len(tx.actions) - 1; i >= 0; i-- {
a := tx.actions[i]
tx.logger.Printf("[INFO] [storage] Rollback: %s", a.description)
if err := a.undo(); err != nil {
tx.logger.Printf("[ERROR] [storage] Rollback failed: %s: %v", a.description, err)
}
}
}
// MigrateDrive performs a full drive migration, moving all apps from source to dest.
func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateRequest, progress chan<- DriveMigrateProgress) error {
start := time.Now()
// TODO: debug should be driven by a dedicated Debug field, not Logger presence.
// Currently always true when Logger is set (which is always in practice).
debug := dm.Logger != nil
dbg := func(format string, args ...interface{}) {
if debug {
dm.Logger.Printf("[DEBUG] [storage] MigrateDrive: "+format, args...)
}
}
_ = dbg // used below
dbg("starting drive migration: source=%s dest=%s", req.SourcePath, req.DestPath)
send := func(step, msg string, pct int) {
progress <- DriveMigrateProgress{
Step: step,
Message: msg,
Percent: pct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
sendDetail := func(step, msg, detail string, pct int) {
progress <- DriveMigrateProgress{
Step: step,
Message: msg,
Detail: detail,
Percent: pct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
fail := func(msg string, err error) error {
errStr := ""
if err != nil {
errStr = err.Error()
}
progress <- DriveMigrateProgress{
Step: "error",
Message: msg,
Error: errStr,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
return fmt.Errorf("%s: %w", msg, err)
}
// Acquire global migration lock
dm.mu.Lock()
if dm.active {
dm.mu.Unlock()
return fail("Egy másik meghajtó-migráció folyamatban van", fmt.Errorf("migration already active"))
}
dm.active = true
dm.mu.Unlock()
defer func() {
dm.mu.Lock()
dm.active = false
dm.mu.Unlock()
}()
// --- Pre-validation ---
send("validating", "Ellenőrzés...", 1)
srcLabel := dm.Sett.GetStorageLabel(req.SourcePath)
dstLabel := dm.Sett.GetStorageLabel(req.DestPath)
if dm.Sett.IsDisconnected(req.SourcePath) {
return fail("A forrás meghajtó le van választva", fmt.Errorf("source disconnected"))
}
if dm.Sett.IsDecommissioned(req.SourcePath) {
return fail("A forrás meghajtó már kiváltott", fmt.Errorf("source decommissioned"))
}
if dm.Sett.IsDisconnected(req.DestPath) {
return fail("A cél meghajtó le van választva", fmt.Errorf("dest disconnected"))
}
if dm.Sett.IsDecommissioned(req.DestPath) {
return fail("A cél meghajtó már kiváltott", fmt.Errorf("dest decommissioned"))
}
// Find apps on source drive
var appsToMigrate []StackSummaryForMigration
for _, stack := range dm.StackProvider.ListDeployedStacks() {
hddPath := dm.StackProvider.GetStackHDDPath(stack.Name)
if hddPath == req.SourcePath {
appsToMigrate = append(appsToMigrate, stack)
}
}
dbg("found %d apps on source drive: %v", len(appsToMigrate), func() []string {
names := make([]string, len(appsToMigrate))
for i, a := range appsToMigrate {
names[i] = a.Name
}
return names
}())
if len(appsToMigrate) == 0 {
return fail("A forrás meghajtón nincs telepített alkalmazás", fmt.Errorf("no apps on source"))
}
// Check for conflicts on destination
for _, app := range appsToMigrate {
destAppData := appbackup.AppDataDir(req.DestPath, app.Name)
if info, err := os.Stat(destAppData); err == nil && info.IsDir() {
entries, _ := os.ReadDir(destAppData)
if len(entries) > 0 {
return fail(
fmt.Sprintf("A cél meghajtón már létezik adat: %s/%s", req.DestPath, app.Name),
fmt.Errorf("conflict: %s already exists on destination", app.Name),
)
}
}
}
// Estimate total size (exclude restic repos inside felhom-data/backups/)
var totalBytes int64
entries, _ := os.ReadDir(req.SourcePath)
for _, entry := range entries {
if !entry.IsDir() {
continue
}
entryPath := filepath.Join(req.SourcePath, entry.Name())
if entry.Name() == appbackup.FelhomDataDir {
// Scan inside namespace dir, excluding restic repos from estimate
subEntries, _ := os.ReadDir(entryPath)
for _, sub := range subEntries {
if !sub.IsDir() {
continue
}
subPath := filepath.Join(entryPath, sub.Name())
if sub.Name() == "backups" {
totalBytes += dirSizeExcluding(subPath, "restic")
} else {
totalBytes += dirSize(subPath)
}
}
} else {
totalBytes += dirSize(entryPath)
}
}
// Check free space on destination
freeBytes := getFreeBytes(req.DestPath)
if freeBytes > 0 && totalBytes > 0 && int64(float64(totalBytes)*1.05) > freeBytes {
return fail(
fmt.Sprintf("Nincs elég szabad hely: szükséges ~%s, szabad %s",
bytesHuman(totalBytes), bytesHuman(freeBytes)),
fmt.Errorf("insufficient disk space"),
)
}
dbg("estimated data: %s (%d bytes), free on dest: %s (%d bytes)", bytesHuman(totalBytes), totalBytes, bytesHuman(freeBytes), freeBytes)
dm.Logger.Printf("[INFO] [storage] Drive migration: %s (%s) → %s (%s), %d apps, ~%s data",
req.SourcePath, srcLabel, req.DestPath, dstLabel, len(appsToMigrate), bytesHuman(totalBytes))
tx := &migrationTx{logger: dm.Logger}
// --- Step 1: Stop all affected apps ---
send("stopping", fmt.Sprintf("Alkalmazások leállítása (%d db)...", len(appsToMigrate)), 5)
var stoppedApps []string
for _, app := range appsToMigrate {
sendDetail("stopping", "Leállítás: "+app.DisplayName, app.Name, 5)
if err := dm.StackProvider.StopStack(app.Name); err != nil {
dm.Logger.Printf("[ERROR] [storage] Drive migration: failed to stop %s: %v", app.Name, err)
// Rollback: restart already stopped apps
send("rolling_back", "Hiba a leállításnál, visszagörgetés...", 0)
for _, name := range stoppedApps {
_ = dm.StackProvider.StartStack(name)
}
return fail("Alkalmazás leállítása sikertelen: "+app.DisplayName, err)
}
stoppedApps = append(stoppedApps, app.Name)
}
tx.add("Restart all stopped apps", func() error {
for _, name := range stoppedApps {
if err := dm.StackProvider.StartStack(name); err != nil {
dm.Logger.Printf("[WARN] [storage] Rollback: failed to restart %s: %v", name, err)
}
}
return nil
})
// --- Step 2: rsync entire drive (excluding restic repos) ---
send("copying", "Adatok másolása...", 10)
rsyncCmd := exec.CommandContext(ctx, "rsync", "-a", "--info=progress2",
"--exclude=felhom-data/backups/primary/restic/",
"--exclude=felhom-data/backups/secondary/restic/",
req.SourcePath+"/", req.DestPath+"/",
)
stdout, err := rsyncCmd.StdoutPipe()
if err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync pipe hiba", err)
}
stderr, err := rsyncCmd.StderrPipe()
if err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync stderr pipe hiba", err)
}
if err := rsyncCmd.Start(); err != nil {
send("rolling_back", "rsync indítása sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("rsync indítás sikertelen", err)
}
// Parse rsync progress
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
if b, pct, ok := parseRsyncProgress(line); ok {
scaledPct := 10 + pct*50/100 // scale to 10-60%
if scaledPct > 60 {
scaledPct = 60
}
progress <- DriveMigrateProgress{
Step: "copying",
Message: fmt.Sprintf("Adatok másolása... %s / %s", bytesHuman(b), bytesHuman(totalBytes)),
BytesCopied: b,
BytesTotal: totalBytes,
Percent: scaledPct,
ElapsedSeconds: int(time.Since(start).Seconds()),
}
}
}
}()
var stderrBuf strings.Builder
var stderrWg sync.WaitGroup
stderrWg.Add(1)
go func() {
defer stderrWg.Done()
buf := make([]byte, 4096)
for {
n, err := stderr.Read(buf)
if n > 0 {
stderrBuf.Write(buf[:n])
}
if err != nil {
break
}
}
}()
if err := rsyncCmd.Wait(); err != nil {
stderrWg.Wait()
dbg("rsync failed after %s: %v — stderr: %s", time.Since(start).Round(time.Second), err, stderrBuf.String())
send("rolling_back", "rsync sikertelen, visszagörgetés...", 0)
tx.rollback()
return fail("Adatmásolás sikertelen", fmt.Errorf("rsync failed: %w — %s", err, stderrBuf.String()))
}
stderrWg.Wait()
dbg("rsync completed in %s", time.Since(start).Round(time.Second))
// --- Step 3: Verify copy ---
send("verifying", "Másolat ellenőrzése...", 62)
for _, app := range appsToMigrate {
destAppData := appbackup.AppDataDir(req.DestPath, app.Name)
if _, err := os.Stat(destAppData); os.IsNotExist(err) {
// appdata might not exist for all apps (SSD-only apps that share the drive)
// Only warn, don't fail
dm.Logger.Printf("[WARN] [storage] Drive migration: %s not found on destination (may be SSD-only)", destAppData)
}
}
// --- Step 4: Update all app configs ---
send("configuring", "Konfiguráció frissítése...", 65)
dbg("updating HDD_PATH for %d apps", len(appsToMigrate))
var configuredApps []string
for i, app := range appsToMigrate {
// Guard: verify app still exists
if !dm.StackProvider.StackExists(app.Name) {
dm.Logger.Printf("[WARN] [storage] Drive migration: app %s no longer exists, skipping config update", app.Name)
continue
}
pct := 65 + (i * 10 / len(appsToMigrate))
sendDetail("configuring", "Konfiguráció: "+app.DisplayName, app.Name, pct)
oldPath := dm.StackProvider.GetStackHDDPath(app.Name)
if err := dm.StackProvider.UpdateStackHDDPath(app.Name, req.DestPath); err != nil {
dm.Logger.Printf("[ERROR] [storage] Drive migration: failed to update HDD_PATH for %s: %v", app.Name, err)
send("rolling_back", "Konfiguráció frissítése sikertelen, visszagörgetés...", 0)
// Rollback config changes
for _, name := range configuredApps {
_ = dm.StackProvider.UpdateStackHDDPath(name, req.SourcePath)
}
tx.rollback()
return fail("HDD_PATH frissítés sikertelen: "+app.DisplayName, err)
}
configuredApps = append(configuredApps, app.Name)
tx.add("Revert HDD_PATH for "+app.Name, func() error {
return dm.StackProvider.UpdateStackHDDPath(app.Name, oldPath)
})
}
// --- Step 5: Update storage registry ---
send("configuring", "Tárolóregiszter frissítése...", 76)
// Transfer IsDefault
allPaths := dm.Sett.GetStoragePaths()
var srcWasDefault bool
var srcWasSchedulable bool
for _, sp := range allPaths {
if sp.Path == req.SourcePath {
srcWasDefault = sp.IsDefault
srcWasSchedulable = sp.Schedulable
}
}
if srcWasDefault {
_ = dm.Sett.SetDefaultStoragePath(req.DestPath)
}
if srcWasSchedulable {
_ = dm.Sett.SetSchedulable(req.DestPath, true)
}
// Mark source as decommissioned
if err := dm.Sett.SetDecommissioned(req.SourcePath, req.DestPath); err != nil {
dm.Logger.Printf("[WARN] [storage] Drive migration: failed to mark source as decommissioned: %v", err)
}
tx.add("Clear decommissioned on source", func() error {
return dm.Sett.ClearDecommissioned(req.SourcePath)
})
// --- Step 6: Update Tier 2 cross-drive configs ---
send("configuring", "Mentési beállítások frissítése...", 78)
allCrossConfigs := dm.Sett.GetAllCrossDriveConfigs()
for name, cfg := range allCrossConfigs {
if cfg == nil {
continue
}
// Apps that moved (source→dest) with Tier 2 pointing to dest: clear (no redundancy)
appHDD := dm.StackProvider.GetStackHDDPath(name)
if appHDD == req.DestPath && cfg.DestinationPath == req.DestPath {
dm.Logger.Printf("[INFO] [storage] Drive migration: clearing Tier 2 for %s (dest same as app drive)", name)
_ = dm.Sett.SetCrossDriveConfig(name, nil)
continue
}
// Apps on OTHER drives with Tier 2 pointing to source: redirect to dest
if cfg.DestinationPath == req.SourcePath {
dm.Logger.Printf("[INFO] [storage] Drive migration: redirecting Tier 2 for %s from %s to %s", name, req.SourcePath, req.DestPath)
cfg.DestinationPath = req.DestPath
_ = dm.Sett.SetCrossDriveConfig(name, cfg)
}
}
// --- Step 7: Start all migrated apps ---
send("starting", "Alkalmazások indítása...", 80)
for i, app := range appsToMigrate {
if !dm.StackProvider.StackExists(app.Name) {
continue
}
pct := 80 + (i * 8 / len(appsToMigrate))
sendDetail("starting", "Indítás: "+app.DisplayName, app.Name, pct)
if err := dm.StackProvider.StartStack(app.Name); err != nil {
dm.Logger.Printf("[WARN] [storage] Drive migration: failed to start %s after migration: %v", app.Name, err)
// Non-fatal — log but continue
}
}
// At this point, migration is considered successful — no more rollback.
// --- Step 8: Trigger immediate backup ---
send("backup", "Biztonsági mentés indítása...", 90)
if dm.BackupTrigger != nil {
if err := dm.BackupTrigger.TryRunDriveBackup(ctx, req.DestPath); err != nil {
dm.Logger.Printf("[WARN] [storage] Drive migration: post-migration backup failed: %v", err)
}
}
// --- Step 9: Post-migration notifications ---
send("configuring", "Befejező lépések...", 95)
if dm.SyncFBMounts != nil {
dm.SyncFBMounts()
}
if dm.AlertRefresh != nil {
dm.AlertRefresh()
}
if dm.PushHubReport != nil {
dm.PushHubReport()
}
if dm.PushInfraBackup != nil {
dm.PushInfraBackup()
}
elapsed := time.Since(start)
dm.Logger.Printf("[INFO] [storage] Drive migration complete: %s → %s, %d apps, %s elapsed",
req.SourcePath, req.DestPath, len(appsToMigrate), elapsed.Round(time.Second))
// --- Step 10: Done ---
appNames := make([]string, len(appsToMigrate))
for i, app := range appsToMigrate {
appNames[i] = app.DisplayName
}
progress <- DriveMigrateProgress{
Step: "done",
Message: fmt.Sprintf("A %s meghajtó sikeresen kiváltva! %d alkalmazás átköltöztetve ide: %s (%s). Idő: %s",
srcLabel, len(appsToMigrate), dstLabel, req.DestPath, elapsed.Round(time.Second)),
Percent: 100,
ElapsedSeconds: int(elapsed.Seconds()),
Detail: strings.Join(appNames, ", "),
}
return nil
}
// dirSizeExcluding returns the total bytes in a directory, excluding subdirectories named excludeName.
func dirSizeExcluding(path, excludeName string) int64 {
var total int64
filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if info.IsDir() && info.Name() == excludeName {
return filepath.SkipDir
}
if !info.IsDir() {
total += info.Size()
}
return nil
})
return total
}
controller/internal/storage/safety.go
-43
View File
@@ -1,43 +0,0 @@
package storage
import (
"fmt"
"regexp"
"strings"
)
// mountNameRe validates mount names: only alphanumeric + underscore.
var mountNameRe = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
// FstabPath is the path to the host fstab inside the container.
// The compose file mounts /etc/fstab → /host-fstab.
const FstabPath = "/host-fstab"
// HostDevPath is where the host's /dev is mounted inside the container.
// Docker always creates its own tmpfs at /dev (overriding any bind mount),
// so the host's block devices are mounted at a different path instead.
const HostDevPath = "/host-dev"
// HostDevicePath converts a standard /dev/ device path to the container-accessible path.
// "/dev/sdb" → "/host-dev/sdb", "/dev/sdb1" → "/host-dev/sdb1"
// Paths that don't start with /dev/ are returned unchanged.
func HostDevicePath(devPath string) string {
if strings.HasPrefix(devPath, "/dev/") {
return HostDevPath + "/" + strings.TrimPrefix(devPath, "/dev/")
}
return devPath
}
// ValidateMountName returns an error if the mount name is invalid.
func ValidateMountName(name string) error {
if name == "" {
return fmt.Errorf("a csatlakoztatási névnek nem szabad üresnek lennie")
}
if !mountNameRe.MatchString(name) {
return fmt.Errorf("a csatlakoztatási néven csak betűk, számok és alávonás megengedett")
}
if len(name) > 32 {
return fmt.Errorf("a csatlakoztatási néven legfeljebb 32 karakter megengedett")
}
return nil
}
controller/internal/storage/safety_linux.go
-222
View File
@@ -1,222 +0,0 @@
//go:build linux
package storage
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"golang.org/x/sys/unix"
)
// IsSystemDisk checks if the given device path overlaps with the root filesystem device.
// Returns true if the device is (or is the parent of) the system disk.
func IsSystemDisk(devicePath string) (bool, error) {
// Get the block device major number of the root filesystem
var rootStat syscall.Stat_t
if err := syscall.Stat("/", &rootStat); err != nil {
return false, fmt.Errorf("cannot stat /: %w", err)
}
// Get block device info of the target device (via /host-dev — Docker overrides /dev)
var devStat syscall.Stat_t
if err := syscall.Stat(HostDevicePath(devicePath), &devStat); err != nil {
return false, fmt.Errorf("cannot stat %s: %w", devicePath, err)
}
// C5: Use unix.Major/Minor for correct 12-bit extraction (old 0xff mask truncated high bits).
// Also compare the disk portion of the minor to distinguish separate physical disks of the
// same type (e.g., sda and sdb both have major 8, but different disk-minor groups of 16).
rootMajor := unix.Major(rootStat.Dev)
rootMinor := unix.Minor(rootStat.Dev)
devMajor := unix.Major(devStat.Rdev)
devMinor := unix.Minor(devStat.Rdev)
if rootMajor != devMajor {
return false, nil
}
// Same major — compare disk groups (each disk gets 16 minor numbers on SCSI/SATA,
// e.g., sda=0-15, sdb=16-31; NVMe uses similar grouping).
rootDiskGroup := rootMinor / 16
devDiskGroup := devMinor / 16
return rootDiskGroup == devDiskGroup, nil
}
// IsSystemPartition checks if a specific partition is a system partition
// (/, /boot, /boot/efi, swap) or is currently mounted.
// Unlike IsSystemDisk() which blocks the entire disk, this checks only the
// individual partition — allowing non-system partitions on system disks to be formatted.
func IsSystemPartition(partitionPath string) (bool, error) {
fstabPath := "/host-fstab"
if _, err := os.Stat(fstabPath); err != nil {
fstabPath = "/etc/fstab"
}
data, err := os.ReadFile(fstabPath)
if err != nil {
// If we can't read fstab, err on the side of caution
return true, fmt.Errorf("cannot read fstab: %w", err)
}
systemMounts := map[string]bool{"/": true, "/boot": true, "/boot/efi": true}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
source := fields[0]
mountPoint := fields[1]
fsType := fields[2]
if !systemMounts[mountPoint] && fsType != "swap" {
continue
}
var devPath string
if strings.HasPrefix(source, "UUID=") {
uuid := strings.TrimPrefix(source, "UUID=")
if out, err := exec.Command("blkid", "-U", uuid).Output(); err == nil {
devPath = strings.TrimSpace(string(out))
}
} else if strings.HasPrefix(source, "/dev/") {
devPath = source
}
if devPath == partitionPath {
return true, nil
}
}
// Also check if the partition is currently mounted
mounted, err := IsDeviceMounted(partitionPath)
if err != nil {
return false, err
}
if mounted {
return true, nil
}
return false, nil
}
// IsDeviceMounted checks if a device or any of its partitions is currently mounted.
func IsDeviceMounted(devicePath string) (bool, error) {
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false, fmt.Errorf("cannot read /proc/mounts: %w", err)
}
base := filepath.Base(devicePath)
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
dev := fields[0]
devBase := filepath.Base(dev)
// H9: Require exact match or that the suffix after base is a digit or 'p' (partition marker).
// Prevents /dev/sdb matching /dev/sdba (hypothetical device) or /dev/sdb_backup (bind).
if devBase == base {
return true, nil
}
if strings.HasPrefix(devBase, base) {
next := devBase[len(base)]
if next >= '0' && next <= '9' || next == 'p' {
return true, nil
}
}
}
return false, nil
}
// IsMountPathInUse checks if a path is already used as a mount point.
func IsMountPathInUse(mountPath string) (bool, error) {
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false, fmt.Errorf("cannot read /proc/mounts: %w", err)
}
mountPath = filepath.Clean(mountPath)
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
if filepath.Clean(fields[1]) == mountPath {
return true, nil
}
}
return false, nil
}
// BackupFstab creates a dated backup of the fstab file.
func BackupFstab(fstabPath string) error {
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("cannot read %s: %w", fstabPath, err)
}
backupPath := fstabPath + ".bak." + time.Now().Format("20060102")
return os.WriteFile(backupPath, data, 0644)
}
// AppendFstabEntry appends a UUID-based fstab entry.
// Uses atomic rename when possible, falls back to direct overwrite for bind-mounted files
// (Docker mounts /etc/fstab as /host-fstab — rename fails with EBUSY on bind mounts).
func AppendFstabEntry(fstabPath, uuid, mountPoint, fsType, options string) error {
existing, err := os.ReadFile(fstabPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("cannot read fstab: %w", err)
}
entry := fmt.Sprintf("\nUUID=%s\t%s\t%s\t%s\t0 2\n", uuid, mountPoint, fsType, options)
newContent := append(existing, []byte(entry)...)
return safeWriteFile(fstabPath, newContent, 0644)
}
// RemoveFstabEntry removes any line containing the given UUID from fstab.
// H19: Called as rollback if mount fails after fstab was written.
func RemoveFstabEntry(fstabPath, uuid string) error {
data, err := os.ReadFile(fstabPath)
if err != nil {
return fmt.Errorf("cannot read fstab: %w", err)
}
var kept []string
for _, line := range strings.Split(string(data), "\n") {
if !strings.Contains(line, "UUID="+uuid) {
kept = append(kept, line)
}
}
newContent := strings.Join(kept, "\n")
return safeWriteFile(fstabPath, []byte(newContent), 0644)
}
// safeWriteFile writes content to a file. It tries atomic rename first (write to .tmp,
// then rename). If rename fails (e.g., bind-mounted files where rename returns EBUSY),
// it falls back to a direct truncate-and-write to the target file.
func safeWriteFile(path string, content []byte, perm os.FileMode) error {
tmpPath := path + ".tmp"
if err := os.WriteFile(tmpPath, content, perm); err != nil {
return fmt.Errorf("cannot write tmp file %s: %w", tmpPath, err)
}
if err := os.Rename(tmpPath, path); err == nil {
return nil // atomic rename succeeded
}
// Rename failed (likely bind mount) — fall back to direct write
os.Remove(tmpPath)
if err := os.WriteFile(path, content, perm); err != nil {
return fmt.Errorf("cannot write %s: %w", path, err)
}
return nil
}
controller/internal/storage/safety_other.go
-29
View File
@@ -1,29 +0,0 @@
//go:build !linux
package storage
import "fmt"
func IsSystemDisk(devicePath string) (bool, error) {
return false, fmt.Errorf("storage init is only supported on Linux")
}
func IsSystemPartition(partitionPath string) (bool, error) {
return false, fmt.Errorf("storage init is only supported on Linux")
}
func IsDeviceMounted(devicePath string) (bool, error) {
return false, fmt.Errorf("storage init is only supported on Linux")
}
func IsMountPathInUse(mountPath string) (bool, error) {
return false, fmt.Errorf("storage init is only supported on Linux")
}
func BackupFstab(fstabPath string) error {
return fmt.Errorf("storage init is only supported on Linux")
}
func AppendFstabEntry(fstabPath, uuid, mountPoint, fsType, options string) error {
return fmt.Errorf("storage init is only supported on Linux")
}
controller/internal/storage/scan.go
-42
View File
@@ -1,42 +0,0 @@
package storage
// BlockDevice represents a detected physical disk.
type BlockDevice struct {
Name string // "sdb"
Path string // "/dev/sdb"
Size string // "931.5G"
SizeBytes int64 // raw bytes from lsblk
Model string // "WD Elements 25A2"
Type string // "disk"
Removable bool // true for USB
Partitions []Partition // child partitions
Mounted bool // any partition is mounted
}
// Partition represents a partition on a block device.
type Partition struct {
Name string // "sdb1"
Path string // "/dev/sdb1"
Size string // "931.5G"
SizeBytes int64
FSType string // "ext4", "" for no filesystem
Label string // filesystem label
UUID string
MountPoint string // "" if not mounted
}
// FormatablePartition is an empty partition on a system disk that can be formatted.
type FormatablePartition struct {
Partition
ParentDiskName string // "sda"
ParentDiskPath string // "/dev/sda"
ParentDiskModel string // "Samsung SSD 870"
ParentDiskSize string // "500 GB"
}
// ScanResult from disk detection.
type ScanResult struct {
AvailableDisks []BlockDevice // Unmounted, non-system disks
SystemDisks []BlockDevice // System/mounted disks (display only)
FormatablePartitions []FormatablePartition // Empty partitions on system disks, safe to format
}
controller/internal/storage/scan_linux.go
-428
View File
@@ -1,428 +0,0 @@
//go:build linux
package storage
import (
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// lsblkOutput matches the top-level JSON from lsblk -J.
type lsblkOutput struct {
BlockDevices []lsblkDevice `json:"blockdevices"`
}
// lsblkDevice is the raw JSON structure from lsblk.
type lsblkDevice struct {
Name string `json:"name"`
Path string `json:"path"`
Size interface{} `json:"size"` // may be float64 or string
Type string `json:"type"`
FSType *string `json:"fstype"`
MountPoint *string `json:"mountpoint"`
Model *string `json:"model"`
RM interface{} `json:"rm"` // removable: bool or "0"/"1"
Children []lsblkDevice `json:"children"`
}
func (d *lsblkDevice) sizeBytes() int64 {
switch v := d.Size.(type) {
case float64:
return int64(v)
case string:
// M3: lsblk can return size as a string on some kernel versions.
n, _ := strconv.ParseUint(v, 10, 64)
return int64(n)
}
return 0
}
func (d *lsblkDevice) sizeHuman() string {
bytes := d.sizeBytes()
const (
GB = 1024 * 1024 * 1024
TB = GB * 1024
)
switch {
case bytes >= TB:
return fmt.Sprintf("%.1f TB", float64(bytes)/float64(TB))
case bytes >= GB:
return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB))
default:
return fmt.Sprintf("%d MB", bytes/(1024*1024))
}
}
func (d *lsblkDevice) isRemovable() bool {
switch v := d.RM.(type) {
case bool:
return v
case float64:
return v != 0
case string:
return v == "1" || strings.EqualFold(v, "true")
}
return false
}
func (d *lsblkDevice) fsType() string {
if d.FSType != nil {
return *d.FSType
}
return ""
}
func (d *lsblkDevice) mountPoint() string {
if d.MountPoint != nil {
return *d.MountPoint
}
return ""
}
func (d *lsblkDevice) model() string {
if d.Model != nil {
return strings.TrimSpace(*d.Model)
}
return ""
}
// getSystemDiskNames returns the set of parent disk names (e.g., "sda")
// that contain system partitions (/, /boot, /boot/efi, swap).
// It reads the host's fstab (mounted at /host-fstab in the container)
// and resolves UUIDs to device paths via blkid.
func getSystemDiskNames() map[string]bool {
systemDisks := map[string]bool{}
// Step 1: Find and parse fstab
fstabPath := "/host-fstab"
if _, err := os.Stat(fstabPath); err != nil {
fstabPath = "/etc/fstab"
}
data, err := os.ReadFile(fstabPath)
if err != nil {
return systemDisks
}
systemMounts := map[string]bool{"/": true, "/boot": true, "/boot/efi": true}
var systemUUIDs []string
var systemDevices []string
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
source := fields[0]
mountPoint := fields[1]
fsType := fields[2]
if !systemMounts[mountPoint] && fsType != "swap" {
continue
}
if strings.HasPrefix(source, "UUID=") {
systemUUIDs = append(systemUUIDs, strings.TrimPrefix(source, "UUID="))
} else if strings.HasPrefix(source, "/dev/") {
systemDevices = append(systemDevices, source)
}
}
// Step 2: Resolve UUIDs to device paths via blkid
for _, uuid := range systemUUIDs {
out, err := exec.Command("blkid", "-U", uuid).Output()
if err == nil {
devPath := strings.TrimSpace(string(out))
if devPath != "" {
systemDevices = append(systemDevices, devPath)
}
}
}
// Step 3: Extract parent disk names from device paths
for _, devPath := range systemDevices {
diskName := partitionToParentDisk(devPath)
if diskName != "" {
systemDisks[diskName] = true
}
}
return systemDisks
}
// getSystemPartitionPaths returns the set of partition device paths (e.g., "/dev/sda3")
// that are system partitions (/, /boot, /boot/efi, swap).
// Unlike getSystemDiskNames() which returns parent disk names, this returns the actual
// partition paths for granular checks.
func getSystemPartitionPaths() map[string]bool {
sysPartitions := map[string]bool{}
fstabPath := "/host-fstab"
if _, err := os.Stat(fstabPath); err != nil {
fstabPath = "/etc/fstab"
}
data, err := os.ReadFile(fstabPath)
if err != nil {
return sysPartitions
}
systemMounts := map[string]bool{"/": true, "/boot": true, "/boot/efi": true}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
source := fields[0]
mountPoint := fields[1]
fsType := fields[2]
if !systemMounts[mountPoint] && fsType != "swap" {
continue
}
if strings.HasPrefix(source, "UUID=") {
uuid := strings.TrimPrefix(source, "UUID=")
if out, err := exec.Command("blkid", "-U", uuid).Output(); err == nil {
devPath := strings.TrimSpace(string(out))
if devPath != "" {
sysPartitions[devPath] = true
}
}
} else if strings.HasPrefix(source, "/dev/") {
sysPartitions[source] = true
}
}
return sysPartitions
}
// partitionToParentDisk extracts the parent disk name from a partition device path.
// "/dev/sda2" → "sda", "/dev/nvme0n1p2" → "nvme0n1", "/dev/mmcblk0p1" → "mmcblk0"
func partitionToParentDisk(devPath string) string {
name := filepath.Base(devPath)
// H10: Handle mmcblk0p1 and nvme0n1p1 patterns where 'p' separates disk# from partition#.
// The prefix before 'p' must end with a digit (e.g., mmcblk0, nvme0n1) to be a disk number.
if idx := strings.LastIndex(name, "p"); idx > 0 {
prefix := name[:idx]
suffix := name[idx+1:]
if len(suffix) > 0 && suffix[0] >= '0' && suffix[0] <= '9' &&
len(prefix) > 0 && prefix[len(prefix)-1] >= '0' && prefix[len(prefix)-1] <= '9' {
// Verify suffix is all digits (partition number, not part of device name)
allDigits := true
for _, c := range suffix {
if c < '0' || c > '9' {
allDigits = false
break
}
}
if allDigits {
return prefix // e.g., mmcblk0, nvme0n1
}
}
}
// Standard: sda2 → sda, sdb1 → sdb
return strings.TrimRight(name, "0123456789")
}
// enrichWithBlkid fills in missing FSType, UUID, and Label on partitions using blkid.
// Probes each partition individually via /host-dev (Docker overrides /dev with its own
// tmpfs, so the host block devices are accessible at /host-dev instead).
func enrichWithBlkid(disks []BlockDevice, logger *log.Logger, debug bool) {
dbg := func(format string, args ...interface{}) {
if debug && logger != nil {
logger.Printf("[DEBUG] [storage] enrichWithBlkid: "+format, args...)
}
}
for i := range disks {
for j := range disks[i].Partitions {
p := &disks[i].Partitions[j]
hostPath := HostDevicePath(p.Path) // "/dev/sdb1" → "/host-dev/sdb1"
if p.FSType == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "TYPE", hostPath).Output(); err == nil {
p.FSType = strings.TrimSpace(string(out))
dbg("blkid TYPE %s → %q", hostPath, p.FSType)
} else {
dbg("blkid TYPE %s failed: %v", hostPath, err)
}
}
if p.UUID == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "UUID", hostPath).Output(); err == nil {
p.UUID = strings.TrimSpace(string(out))
dbg("blkid UUID %s → %q", hostPath, p.UUID)
} else {
dbg("blkid UUID %s failed: %v", hostPath, err)
}
}
if p.Label == "" {
if out, err := exec.Command("blkid", "-o", "value", "-s", "LABEL", hostPath).Output(); err == nil {
p.Label = strings.TrimSpace(string(out))
dbg("blkid LABEL %s → %q", hostPath, p.Label)
} else {
dbg("blkid LABEL %s failed: %v", hostPath, err)
}
}
}
}
}
// ScanDisks detects all block devices and classifies them into
// available (not mounted, not system) and system/mounted disks.
func ScanDisks(logger *log.Logger, debug bool) (*ScanResult, error) {
dbg := func(format string, args ...interface{}) {
if debug && logger != nil {
logger.Printf("[DEBUG] [storage] ScanDisks: "+format, args...)
}
}
if logger != nil {
logger.Printf("[INFO] [storage] Scanning disks")
}
dbg("starting disk scan")
scanStart := time.Now()
out, err := exec.Command(
"lsblk", "-J", "-b",
"-o", "NAME,PATH,SIZE,TYPE,FSTYPE,MOUNTPOINT,MODEL,RM",
).Output()
if err != nil {
return nil, fmt.Errorf("lsblk failed: %w", err)
}
dbg("raw lsblk JSON: %s", util.TruncateStr(string(out), 500))
var parsed lsblkOutput
if err := json.Unmarshal(out, &parsed); err != nil {
return nil, fmt.Errorf("lsblk JSON parse failed: %w", err)
}
dbg("lsblk returned %d block devices", len(parsed.BlockDevices))
// Get system disk names from host fstab (works correctly inside container)
systemDiskNames := getSystemDiskNames()
result := &ScanResult{}
for _, dev := range parsed.BlockDevices {
if dev.Type != "disk" {
continue
}
bd := BlockDevice{
Name: dev.Name,
Path: dev.Path,
Size: dev.sizeHuman(),
SizeBytes: dev.sizeBytes(),
Model: dev.model(),
Type: dev.Type,
Removable: dev.isRemovable(),
}
if bd.Path == "" {
bd.Path = "/dev/" + bd.Name
}
anyMounted := false
for _, child := range dev.Children {
if child.Type != "part" && child.Type != "lvm" && child.Type != "crypt" {
continue
}
part := Partition{
Name: child.Name,
Path: child.Path,
Size: child.sizeHuman(),
SizeBytes: child.sizeBytes(),
FSType: child.fsType(),
MountPoint: child.mountPoint(),
}
if part.Path == "" {
part.Path = "/dev/" + part.Name
}
bd.Partitions = append(bd.Partitions, part)
if part.MountPoint != "" {
anyMounted = true
}
}
// Also check if the disk itself is directly mounted (no partition table)
if dev.mountPoint() != "" {
anyMounted = true
}
isSystem := systemDiskNames[dev.Name]
bd.Mounted = anyMounted || isSystem
classification := "available"
if isSystem || anyMounted {
classification = "system"
result.SystemDisks = append(result.SystemDisks, bd)
} else {
result.AvailableDisks = append(result.AvailableDisks, bd)
}
dbg("disk %s: model=%q size=%s partitions=%d removable=%v classification=%s",
bd.Name, bd.Model, bd.Size, len(bd.Partitions), bd.Removable, classification)
for _, p := range bd.Partitions {
dbg(" partition %s: fstype=%q mountpoint=%q size=%s", p.Name, p.FSType, p.MountPoint, p.Size)
}
}
dbg("classification result: %d available, %d system", len(result.AvailableDisks), len(result.SystemDisks))
// Enrich FSType, UUID, Label from blkid (lsblk can't probe fstype in container)
enrichWithBlkid(result.AvailableDisks, logger, debug)
enrichWithBlkid(result.SystemDisks, logger, debug)
// Detect formatable partitions on system disks:
// empty (no filesystem), unmounted, and NOT a system partition.
sysPartitionPaths := getSystemPartitionPaths()
for _, sysDisk := range result.SystemDisks {
for _, part := range sysDisk.Partitions {
if part.FSType != "" || part.MountPoint != "" {
continue
}
if sysPartitionPaths[part.Path] {
continue
}
result.FormatablePartitions = append(result.FormatablePartitions, FormatablePartition{
Partition: part,
ParentDiskName: sysDisk.Name,
ParentDiskPath: sysDisk.Path,
ParentDiskModel: sysDisk.Model,
ParentDiskSize: sysDisk.Size,
})
dbg("formatable partition found: %s on system disk %s", part.Path, sysDisk.Name)
}
}
if logger != nil {
logger.Printf("[INFO] [storage] Found %d disks, %d formatable partitions",
len(result.AvailableDisks)+len(result.SystemDisks), len(result.FormatablePartitions))
}
dbg("disk scan completed in %s", time.Since(scanStart).Round(time.Millisecond))
return result, nil
}
controller/internal/storage/scan_other.go
-13
View File
@@ -1,13 +0,0 @@
//go:build !linux
package storage
import (
"fmt"
"log"
)
// ScanDisks is not supported on non-Linux platforms.
func ScanDisks(logger *log.Logger, debug bool) (*ScanResult, error) {
return nil, fmt.Errorf("storage init is only supported on Linux")
}
controller/internal/web/agent_disk_handlers.go
+170
View File
@@ -0,0 +1,170 @@
package web
import (
"encoding/json"
"errors"
"net/http"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
)
// Agent-backed disk management (slice 8C, Phase B.2).
//
// Disk EXECUTION (scan/format/mount/migrate) lives in the host agent now; the
// controller is Docker-only and holds no Proxmox/disk credentials. These handlers
// are THIN proxies: they build an agentapi.Client from cfg.LocalAPI and forward
// list/assign/eject/format to the agent's GET/POST /disks endpoints, returning the
// agent's view as JSON. A data-bearing format is refused by the agent (operator
// authorization required) and surfaced here as HTTP 409.
// ServeDiskAPI dispatches /api/disks and /api/disks/* routes.
// Wired in main.go behind RequireAuth + CsrfProtect.
func (s *Server) ServeDiskAPI(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] ServeDiskAPI: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
}
switch {
case r.URL.Path == "/api/disks" && r.Method == http.MethodGet:
s.agentDisksListHandler(w, r)
case r.URL.Path == "/api/disks/assign" && r.Method == http.MethodPost:
s.agentDiskAssignHandler(w, r)
case r.URL.Path == "/api/disks/eject" && r.Method == http.MethodPost:
s.agentDiskEjectHandler(w, r)
case r.URL.Path == "/api/disks/format" && r.Method == http.MethodPost:
s.agentDiskFormatHandler(w, r)
default:
http.NotFound(w, r)
}
}
// agentClient builds a pinned client for the host agent's per-guest local API.
// Returns a clear error if the local API is not configured (unprovisioned guest).
func (s *Server) agentClient() (*agentapi.Client, error) {
if s.cfg.LocalAPI.Endpoint == "" {
return nil, errors.New("agent not configured")
}
return agentapi.New(s.cfg.LocalAPI.Endpoint, s.cfg.LocalAPI.Token, s.cfg.LocalAPI.Fingerprint)
}
// writeDiskJSON writes the standard {ok,data,error} envelope used by the disk API.
func writeDiskJSON(w http.ResponseWriter, status int, ok bool, errMsg string, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
resp := map[string]interface{}{"ok": ok}
if errMsg != "" {
resp["error"] = errMsg
}
if data != nil {
resp["data"] = data
}
_ = json.NewEncoder(w).Encode(resp)
}
// agentDisksListHandler proxies GET /api/disks → agent GET /disks.
func (s *Server) agentDisksListHandler(w http.ResponseWriter, r *http.Request) {
client, err := s.agentClient()
if err != nil {
writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
return
}
resp, err := client.Disks(r.Context())
if err != nil {
s.logger.Printf("[ERROR] [web] disk list via agent failed: %v", err)
writeDiskJSON(w, http.StatusBadGateway, false, err.Error(), nil)
return
}
writeDiskJSON(w, http.StatusOK, true, "", resp)
}
// agentDiskAssignHandler proxies POST /api/disks/assign → agent POST /disks/assign.
func (s *Server) agentDiskAssignHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
UUID string `json:"uuid"`
Where string `json:"where"`
FSType string `json:"fstype"`
Options string `json:"options"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeDiskJSON(w, http.StatusBadRequest, false, "invalid request body", nil)
return
}
if req.UUID == "" || req.Where == "" {
writeDiskJSON(w, http.StatusBadRequest, false, "uuid and where are required", nil)
return
}
client, err := s.agentClient()
if err != nil {
writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
return
}
if err := client.AssignDisk(r.Context(), req.UUID, req.Where, req.FSType, req.Options); err != nil {
s.logger.Printf("[ERROR] [web] disk assign via agent failed: %v", err)
writeDiskJSON(w, http.StatusBadGateway, false, err.Error(), nil)
return
}
writeDiskJSON(w, http.StatusOK, true, "", map[string]interface{}{
"uuid": req.UUID, "where": req.Where,
})
}
// agentDiskEjectHandler proxies POST /api/disks/eject → agent POST /disks/eject.
func (s *Server) agentDiskEjectHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
Where string `json:"where"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeDiskJSON(w, http.StatusBadRequest, false, "invalid request body", nil)
return
}
if req.Where == "" {
writeDiskJSON(w, http.StatusBadRequest, false, "where is required", nil)
return
}
client, err := s.agentClient()
if err != nil {
writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
return
}
resp, err := client.EjectDisk(r.Context(), req.Where)
if err != nil {
s.logger.Printf("[ERROR] [web] disk eject via agent failed: %v", err)
writeDiskJSON(w, http.StatusBadGateway, false, err.Error(), nil)
return
}
writeDiskJSON(w, http.StatusOK, true, "", resp)
}
// agentDiskFormatHandler proxies POST /api/disks/format → agent POST /disks/format.
// A data-bearing format refusal (ErrFormatRefused) is surfaced as HTTP 409 so the UI
// can show "operator authorization required".
func (s *Server) agentDiskFormatHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
Device string `json:"device"`
FSType string `json:"fstype"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeDiskJSON(w, http.StatusBadRequest, false, "invalid request body", nil)
return
}
if req.Device == "" {
writeDiskJSON(w, http.StatusBadRequest, false, "device is required", nil)
return
}
client, err := s.agentClient()
if err != nil {
writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
return
}
resp, err := client.FormatDisk(r.Context(), req.Device, req.FSType)
if errors.Is(err, agentapi.ErrFormatRefused) {
s.logger.Printf("[WARN] [web] disk format refused by agent (data-bearing): %s", req.Device)
writeDiskJSON(w, http.StatusConflict, false, "operator authorization required", resp)
return
}
if err != nil {
s.logger.Printf("[ERROR] [web] disk format via agent failed: %v", err)
writeDiskJSON(w, http.StatusBadGateway, false, err.Error(), nil)
return
}
writeDiskJSON(w, http.StatusOK, true, "", resp)
}
controller/internal/web/handler_debug.go
+7 -214
View File
@@ -14,7 +14,6 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
@@ -23,13 +22,11 @@ import (
// DebugCallbacks holds functions that need main.go wiring (modules not directly on Server).
type DebugCallbacks struct {
TriggerHubReportPush func() error
TriggerHubInfraPush func() error
TriggerLocalInfraWrite func() error
TriggerSetupMode func() error
HubConnectivityTest func() (statusCode int, latencyMs int64, err error)
GiteaConnectivityTest func() (statusCode int, latencyMs int64, err error)
GetTelemetryPreview func() ([]report.AppTelemetry, error)
TriggerHubReportPush func() error
TriggerSetupMode func() error
HubConnectivityTest func() (statusCode int, latencyMs int64, err error)
GiteaConnectivityTest func() (statusCode int, latencyMs int64, err error)
GetTelemetryPreview func() ([]report.AppTelemetry, error)
}
// debugPageHandler renders the debug dashboard page.
@@ -53,29 +50,13 @@ func (s *Server) handleDebugAPI(w http.ResponseWriter, r *http.Request) {
case subpath == "event/history" && r.Method == http.MethodGet:
s.debugEventHistory(w, r)
// Section 3: Backup testing
// Section 3: Backup testing (app-data only; disk-tier moved to host agent)
case subpath == "backup/dbdump" && r.Method == http.MethodPost:
s.debugTriggerDBDump(w, r)
case subpath == "backup/crossdrive" && r.Method == http.MethodPost:
s.debugTriggerCrossDrive(w, r)
case subpath == "backup/integrity" && r.Method == http.MethodPost:
s.debugTriggerIntegrity(w, r)
case subpath == "backup/infra" && r.Method == http.MethodPost:
s.debugTriggerInfraBackup(w, r)
// Section 4: Storage simulation
case subpath == "storage/simulate-disconnect" && r.Method == http.MethodPost:
s.debugSimulateDisconnect(w, r)
case subpath == "storage/simulate-reconnect" && r.Method == http.MethodPost:
s.debugSimulateReconnect(w, r)
case subpath == "storage/watchdog-status" && r.Method == http.MethodGet:
s.debugWatchdogStatus(w, r)
// Section 5: Hub & connectivity
case subpath == "hub/push" && r.Method == http.MethodPost:
s.debugHubPush(w, r)
case subpath == "hub/infra-push" && r.Method == http.MethodPost:
s.debugHubInfraPush(w, r)
case subpath == "hub/test-connectivity" && r.Method == http.MethodPost:
s.debugHubConnectivity(w, r)
case subpath == "hub/preferences-sync" && r.Method == http.MethodPost:
@@ -94,8 +75,6 @@ func (s *Server) handleDebugAPI(w http.ResponseWriter, r *http.Request) {
// Section 7: DR / Setup
case subpath == "dr/trigger-setup" && r.Method == http.MethodPost:
s.debugTriggerSetupWizard(w, r)
case subpath == "dr/infra-status" && r.Method == http.MethodGet:
s.debugInfraBackupStatus(w, r)
// Section 8: Log viewer
case subpath == "logs" && r.Method == http.MethodGet:
@@ -219,23 +198,13 @@ func (s *Server) debugDump(w http.ResponseWriter, r *http.Request) {
"enabled": true,
"running": s.backupMgr.IsRunning(),
}
dbDump, backupSt := s.backupMgr.GetStatus()
dbDump := s.backupMgr.GetStatus()
if dbDump != nil {
backupInfo["last_db_dump"] = map[string]interface{}{
"time": dbDump.LastRun,
"success": dbDump.Success,
}
}
if backupSt != nil {
backupInfo["last_backup"] = map[string]interface{}{
"time": backupSt.LastRun,
"success": backupSt.Success,
}
if backupSt.RepoStats != nil {
backupInfo["repo_size"] = backupSt.RepoStats.TotalSize
backupInfo["snapshot_count"] = backupSt.RepoStats.SnapshotCount
}
}
dump["backup"] = backupInfo
} else {
dump["backup"] = map[string]interface{}{"enabled": false}
@@ -378,98 +347,6 @@ func (s *Server) debugTriggerDBDump(w http.ResponseWriter, r *http.Request) {
writeDebugJSON(w, http.StatusOK, true, "DB dump elindítva", nil)
}
func (s *Server) debugTriggerCrossDrive(w http.ResponseWriter, r *http.Request) {
if s.crossDriveRunner == nil {
writeDebugJSON(w, http.StatusBadRequest, false, "Cross-drive runner nincs konfigurálva", nil)
return
}
go func() {
if err := s.crossDriveRunner.RunAllConfigured(context.Background()); err != nil {
s.logger.Printf("[WARN] Debug cross-drive failed: %v", err)
}
}()
writeDebugJSON(w, http.StatusOK, true, "Cross-drive mentés elindítva", nil)
}
func (s *Server) debugTriggerIntegrity(w http.ResponseWriter, r *http.Request) {
if s.backupMgr == nil {
writeDebugJSON(w, http.StatusBadRequest, false, "Backup manager nincs konfigurálva", nil)
return
}
go func() {
if err := s.backupMgr.RunIntegrityCheck(context.Background()); err != nil {
s.logger.Printf("[WARN] Debug integrity check failed: %v", err)
}
}()
writeDebugJSON(w, http.StatusOK, true, "Integritás ellenőrzés elindítva", nil)
}
func (s *Server) debugTriggerInfraBackup(w http.ResponseWriter, r *http.Request) {
if s.debugCallbacks == nil || s.debugCallbacks.TriggerLocalInfraWrite == nil {
writeDebugJSON(w, http.StatusNotImplemented, false, "Nem bekötött", nil)
return
}
go func() {
if err := s.debugCallbacks.TriggerLocalInfraWrite(); err != nil {
s.logger.Printf("[WARN] Debug infra backup failed: %v", err)
}
}()
writeDebugJSON(w, http.StatusOK, true, "Infra mentés elindítva", nil)
}
// ── Section 4: Storage simulation ───────────────────────────────────
func (s *Server) debugSimulateDisconnect(w http.ResponseWriter, r *http.Request) {
var req struct {
Path string `json:"path"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Path == "" {
writeDebugJSON(w, http.StatusBadRequest, false, "Érvénytelen kérés: path szükséges", nil)
return
}
if s.storageWatchdog == nil {
writeDebugJSON(w, http.StatusBadRequest, false, "Storage watchdog nincs konfigurálva", nil)
return
}
stopped, err := s.storageWatchdog.SimulateDisconnect(r.Context(), req.Path)
if err != nil {
writeDebugJSON(w, http.StatusBadRequest, false, err.Error(), nil)
return
}
writeDebugJSON(w, http.StatusOK, true,
fmt.Sprintf("Leválasztás szimulálva: %s (%d app leállítva)", req.Path, len(stopped)),
map[string]interface{}{"stopped_stacks": stopped})
}
func (s *Server) debugSimulateReconnect(w http.ResponseWriter, r *http.Request) {
var req struct {
Path string `json:"path"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Path == "" {
writeDebugJSON(w, http.StatusBadRequest, false, "Érvénytelen kérés: path szükséges", nil)
return
}
if s.storageWatchdog == nil {
writeDebugJSON(w, http.StatusBadRequest, false, "Storage watchdog nincs konfigurálva", nil)
return
}
if err := s.storageWatchdog.SimulateReconnect(r.Context(), req.Path); err != nil {
writeDebugJSON(w, http.StatusBadRequest, false, err.Error(), nil)
return
}
writeDebugJSON(w, http.StatusOK, true,
fmt.Sprintf("Visszacsatlakozás szimulálva: %s", req.Path), nil)
}
func (s *Server) debugWatchdogStatus(w http.ResponseWriter, r *http.Request) {
if s.storageWatchdog == nil {
writeDebugJSON(w, http.StatusOK, true, "", []interface{}{})
return
}
status := s.storageWatchdog.GetDebugStatus()
writeDebugJSON(w, http.StatusOK, true, "", status)
}
// ── Section 5: Hub & connectivity ───────────────────────────────────
func (s *Server) debugHubPush(w http.ResponseWriter, r *http.Request) {
@@ -488,22 +365,6 @@ func (s *Server) debugHubPush(w http.ResponseWriter, r *http.Request) {
map[string]interface{}{"latency_ms": latency})
}
func (s *Server) debugHubInfraPush(w http.ResponseWriter, r *http.Request) {
if s.debugCallbacks == nil || s.debugCallbacks.TriggerHubInfraPush == nil {
writeDebugJSON(w, http.StatusNotImplemented, false, "Nem bekötött", nil)
return
}
start := time.Now()
err := s.debugCallbacks.TriggerHubInfraPush()
latency := time.Since(start).Milliseconds()
if err != nil {
writeDebugJSON(w, http.StatusOK, false, err.Error(), map[string]interface{}{"latency_ms": latency})
return
}
writeDebugJSON(w, http.StatusOK, true, "Infra backup elküldve a Hubra",
map[string]interface{}{"latency_ms": latency})
}
func (s *Server) debugHubConnectivity(w http.ResponseWriter, r *http.Request) {
if s.debugCallbacks == nil || s.debugCallbacks.HubConnectivityTest == nil {
writeDebugJSON(w, http.StatusNotImplemented, false, "Nem bekötött", nil)
@@ -613,13 +474,6 @@ func (s *Server) debugTriggerSetupWizard(w http.ResponseWriter, r *http.Request)
return
}
// Pre-check: verify infra backup exists on at least one drive
if !s.hasInfraBackupOnDrive() {
writeDebugJSON(w, http.StatusBadRequest, false,
"Nincs infra backup egyetlen meghajtón sem! Először készítsen infra backupot.", nil)
return
}
// Write marker file
markerPath := filepath.Join(s.cfg.Paths.DataDir, ".needs-setup")
if err := os.WriteFile(markerPath, []byte("debug-triggered\n"), 0644); err != nil {
@@ -637,67 +491,6 @@ func (s *Server) debugTriggerSetupWizard(w http.ResponseWriter, r *http.Request)
}()
}
func (s *Server) debugInfraBackupStatus(w http.ResponseWriter, r *http.Request) {
storagePaths := s.settings.GetStoragePaths()
drives := make([]map[string]interface{}, 0, len(storagePaths))
for _, sp := range storagePaths {
if sp.Decommissioned || sp.Disconnected {
continue
}
driveInfo := map[string]interface{}{
"path": sp.Path,
"label": sp.Label,
"has_backup": false,
}
infraDir := backup.InfraBackupDir(sp.Path)
info, err := os.Stat(infraDir)
if err == nil && info.IsDir() {
driveInfo["has_backup"] = true
driveInfo["last_modified"] = info.ModTime()
// List files
entries, _ := os.ReadDir(infraDir)
files := make([]string, 0, len(entries))
for _, e := range entries {
files = append(files, e.Name())
}
driveInfo["files"] = files
}
drives = append(drives, driveInfo)
}
data := map[string]interface{}{
"drives": drives,
}
if s.hubPushStatusFn != nil {
st := s.hubPushStatusFn()
data["hub_infra_push"] = map[string]interface{}{
"last_attempt": st.LastAttempt,
"last_success": st.LastSuccess,
"last_error": st.LastError,
}
}
writeDebugJSON(w, http.StatusOK, true, "", data)
}
// hasInfraBackupOnDrive checks if any connected storage drive has an infra backup.
func (s *Server) hasInfraBackupOnDrive() bool {
for _, sp := range s.settings.GetStoragePaths() {
if sp.Decommissioned || sp.Disconnected {
continue
}
infraDir := backup.InfraBackupDir(sp.Path)
if info, err := os.Stat(infraDir); err == nil && info.IsDir() {
return true
}
}
return false
}
// ── Section 8: Log viewer ───────────────────────────────────────────
func (s *Server) debugLogBuffer(w http.ResponseWriter, r *http.Request) {
controller/internal/web/handler_export.go
+17
View File
@@ -9,6 +9,23 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
)
// jsonResponse writes a JSON body with a 200 status.
// (Shared JSON helper previously defined in the now-removed storage_handlers.go.)
func jsonResponse(w http.ResponseWriter, v interface{}) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(v)
}
// jsonError writes a JSON error response with the given status code.
func jsonError(w http.ResponseWriter, msg string, code int) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(code)
json.NewEncoder(w).Encode(map[string]interface{}{
"ok": false,
"error": msg,
})
}
// ServeExportAPI dispatches /api/export/* endpoints.
func (s *Server) ServeExportAPI(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
controller/internal/web/handler_restore.go
-206
View File
@@ -1,206 +0,0 @@
package web
import (
"context"
"encoding/json"
"net/http"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
)
// restorePageHandler renders the full-page DR restore UI.
func (s *Server) restorePageHandler(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] restorePageHandler: rendering restore page")
}
s.restoreMu.RLock()
plan := s.restorePlan
if plan == nil {
s.restoreMu.RUnlock()
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] restorePageHandler: no restore plan, redirecting to /")
}
http.Redirect(w, r, "/", http.StatusFound)
return
}
// Snapshot all needed fields under lock before rendering
customerID := plan.CustomerID
timestamp := plan.Timestamp
apps := plan.GetApps()
drives := make([]backup.DriveInfo, len(plan.Drives))
copy(drives, plan.Drives)
status := plan.GetStatus()
s.restoreMu.RUnlock()
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] restorePageHandler: customer=%s apps=%d drives=%d status=%s", customerID, len(apps), len(drives), status)
}
data := map[string]interface{}{
"Title": "Katasztrófa utáni visszaállítás",
"CustomerName": s.cfg.Customer.Name,
"Domain": s.cfg.Customer.Domain,
"Version": s.version,
"CustomerID": customerID,
"Timestamp": timestamp,
"Apps": apps,
"Drives": drives,
"PlanStatus": status,
}
s.executeTemplate(w, r, "restore", data)
}
// apiRestoreStatus returns the current restore plan status as JSON.
func (s *Server) apiRestoreStatus(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] apiRestoreStatus: status poll from %s", r.RemoteAddr)
}
s.restoreMu.RLock()
plan := s.restorePlan
if plan == nil {
s.restoreMu.RUnlock()
jsonError(w, "not in restore mode", http.StatusBadRequest)
return
}
snapshot := plan.Snapshot()
s.restoreMu.RUnlock()
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(snapshot)
}
// apiRestoreAll starts restoring all pending apps sequentially.
func (s *Server) apiRestoreAll(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] apiRestoreAll: restore-all requested from %s", r.RemoteAddr)
}
s.restoreMu.RLock()
plan := s.restorePlan
s.restoreMu.RUnlock()
if plan == nil {
jsonError(w, "not in restore mode", http.StatusBadRequest)
return
}
if !plan.TryStartRestore() {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] apiRestoreAll: restore already in progress, rejecting")
}
jsonError(w, "restore already in progress", http.StatusConflict)
return
}
s.logger.Printf("[INFO] [web] Restore-all initiated from %s", r.RemoteAddr)
go s.executeAllRestores()
jsonResponse(w, map[string]interface{}{
"ok": true,
"message": "Visszaállítás elindítva",
})
}
// apiRestoreSkip exits restore mode without restoring.
func (s *Server) apiRestoreSkip(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] apiRestoreSkip: skip requested from %s", r.RemoteAddr)
}
s.restoreMu.RLock()
plan := s.restorePlan
s.restoreMu.RUnlock()
if plan == nil {
jsonError(w, "not in restore mode", http.StatusBadRequest)
return
}
s.logger.Println("[INFO] [web] User skipped DR restore — entering normal mode")
s.clearRestoreMode()
jsonResponse(w, map[string]interface{}{
"ok": true,
"message": "Visszaállítás kihagyva",
})
}
// executeAllRestores runs the restore for each pending app sequentially.
func (s *Server) executeAllRestores() {
s.logger.Println("[INFO] [web] Starting DR restore for all apps")
restoreStart := time.Now()
s.restoreMu.RLock()
plan := s.restorePlan
s.restoreMu.RUnlock()
if plan == nil {
s.logger.Println("[WARN] [web] Restore plan cleared before execution could start")
return
}
// Count pending apps and push DR start event
pendingCount := 0
for _, app := range plan.Apps {
if app.Status == "pending" {
pendingCount++
}
}
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] executeAllRestores: %d pending apps to restore", pendingCount)
}
if s.notifier != nil {
s.notifier.NotifyDRStarted(pendingCount)
}
successCount, failCount := 0, 0
for i := range plan.Apps {
app := &plan.Apps[i]
if app.Status != "pending" {
continue
}
plan.UpdateApp(app.Name, "restoring", "")
s.logger.Printf("[INFO] [web] Restoring app %s (%s)", app.Name, app.DisplayName)
appStart := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
err := backup.RestoreAppFromBackup(ctx, app, s.cfg.Paths.StacksDir, s.logger)
cancel()
if err != nil {
plan.UpdateApp(app.Name, "failed", err.Error())
s.logger.Printf("[ERROR] [web] Restore failed for %s: %v", app.Name, err)
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] executeAllRestores: app=%s failed after %s", app.Name, time.Since(appStart))
}
failCount++
} else {
plan.UpdateApp(app.Name, "done", "")
s.logger.Printf("[INFO] [web] Restore completed for %s", app.Name)
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] executeAllRestores: app=%s completed in %s", app.Name, time.Since(appStart))
}
successCount++
}
}
plan.SetStatus("done")
s.logger.Println("[INFO] [web] All app restores completed")
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] executeAllRestores: total=%d success=%d fail=%d elapsed=%s", pendingCount, successCount, failCount, time.Since(restoreStart))
}
// Push DR completion event
if s.notifier != nil {
s.notifier.NotifyDRCompleted(successCount, failCount)
}
// Re-scan stacks so dashboard picks up restored apps
if s.stackMgr != nil {
if err := s.stackMgr.ScanStacks(); err != nil {
s.logger.Printf("[WARN] [web] Post-restore stack scan failed: %v", err)
}
}
}
// clearRestoreMode exits restore mode and returns to normal operation.
func (s *Server) clearRestoreMode() {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
s.restorePlan = nil
}
controller/internal/web/handlers.go
+24 -415
View File
@@ -20,7 +20,6 @@ import (
"golang.org/x/crypto/bcrypt"
)
// protectedStackSubdomains maps programmatically managed protected stacks
// to their well-known subdomains (these stacks have no .felhom.yml or app.yaml).
var protectedStackSubdomains = map[string]string{
@@ -29,8 +28,8 @@ var protectedStackSubdomains = map[string]string{
// StorageBarInfo holds data for rendering a storage usage bar on dashboard/monitoring.
type StorageBarInfo struct {
Label string // e.g., "USB HDD 1TB", "SYS Storage 350G"
Path string // e.g., "/mnt/hdd_1"
Label string // e.g., "USB HDD 1TB", "SYS Storage 350G"
Path string // e.g., "/mnt/hdd_1"
TotalGB float64
UsedGB float64
Percent float64
@@ -148,34 +147,10 @@ func (s *Server) dashboardHandler(w http.ResponseWriter, r *http.Request) {
data["BackupEnabled"] = s.cfg.Backup.Enabled
if s.backupMgr != nil {
nextDBDump := scheduler.NextDailyRun(s.cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(s.cfg.Backup.ResticSchedule)
fullStatus := s.backupMgr.GetFullStatus(nextDBDump, nextBackup)
fullStatus := s.backupMgr.GetFullStatus(nextDBDump)
data["DBDumpStatus"] = fullStatus.LastDBDump
data["BackupStatus"] = fullStatus.LastBackup
data["BackupRunning"] = fullStatus.Running
data["BackupMaxAgeHours"] = s.cfg.Monitoring.Thresholds.BackupMaxAgeHours
// Cross-drive summary for dashboard Tier 2 status line
crossConfigs := s.settings.GetAllCrossDriveConfigs()
crossDriveTotal := 0
crossDriveConfigured := 0
crossDriveFailed := 0
for _, st := range deployedStacks {
if st.Protected {
continue
}
crossDriveTotal++
cfg, hasCfg := crossConfigs[st.Name]
if hasCfg && cfg != nil && cfg.Enabled {
crossDriveConfigured++
if cfg.LastStatus == "error" {
crossDriveFailed++
}
}
}
data["CrossDriveTotal"] = crossDriveTotal
data["CrossDriveConfigured"] = crossDriveConfigured
data["CrossDriveFailed"] = crossDriveFailed
}
// Build subdomain map for "Megnyitás" buttons
@@ -350,53 +325,10 @@ func (s *Server) deployHandler(w http.ResponseWriter, r *http.Request, name stri
}
}
// Storage info for already-deployed apps with HDD data
// Disk-tier storage management (drive info, stale-data cleanup, cross-drive
// backup) has moved to the host agent (slice 8C); the deploy page no longer
// renders those sections.
if alreadyDeployed {
storageInfo := s.storageInfoForStack(name)
if storageInfo != nil {
data["StorageInfo"] = storageInfo
data["OtherStoragePaths"] = s.otherStoragePathsForStack(name)
}
// Stale data from previous migrations (only for deployed apps with HDD data)
staleData := s.findStaleStorageData(name)
if len(staleData) > 0 {
data["StaleData"] = staleData
}
// Cross-drive backup config for this app
crossCfg := s.settings.GetCrossDriveConfig(name)
data["CrossDriveConfig"] = crossCfg
// Other storage paths for destination dropdown (exclude the app's current storage path)
currentPath := ""
if storageInfo != nil {
currentPath = storageInfo.Path
}
var destPaths []DeployStoragePath
for _, sp := range s.settings.GetStoragePaths() {
if sp.Path == currentPath {
continue // skip the app's current storage — must be a DIFFERENT physical device
}
dp := DeployStoragePath{StoragePath: sp}
if di := system.GetDiskUsage(sp.Path); di != nil {
dp.FreeHuman = formatFreeSpace(di.AvailGB)
if di.TotalGB > 0 {
dp.FreePercent = di.AvailGB / di.TotalGB * 100
}
}
destPaths = append(destPaths, dp)
}
data["BackupDestPaths"] = destPaths
// Destination health warning (tiered validation)
if crossCfg != nil && crossCfg.Enabled && crossCfg.DestinationPath != "" {
health := system.CheckBackupDestination(crossCfg.DestinationPath)
if health.Warning != "" {
data["BackupDestWarning"] = health.Warning
data["BackupDestWarningSeverity"] = health.Severity
}
}
// App-to-app integrations
if meta.HasIntegrations() && s.integrationMgr != nil {
data["HasIntegrations"] = true
@@ -581,8 +513,7 @@ func (s *Server) backupsHandler(w http.ResponseWriter, r *http.Request) {
if s.backupMgr != nil {
nextDBDump := scheduler.NextDailyRun(s.cfg.Backup.DBDumpSchedule)
nextBackup := scheduler.NextDailyRun(s.cfg.Backup.ResticSchedule)
fullStatus := s.backupMgr.GetFullStatus(nextDBDump, nextBackup)
fullStatus := s.backupMgr.GetFullStatus(nextDBDump)
// Pass flash messages from query params (set by redirect handlers)
if flash := r.URL.Query().Get("flash"); flash != "" {
@@ -608,143 +539,18 @@ func (s *Server) backupsHandler(w http.ResponseWriter, r *http.Request) {
}
}
// Build cross-drive summary
crossConfigs := s.settings.GetAllCrossDriveConfigs()
// Build label lookup for dest paths
destLabels := make(map[string]string)
for _, sp := range storagePaths {
destLabels[sp.Path] = sp.Label
}
for _, app := range fullStatus.AppDataInfo {
cfg, hasCfg := crossConfigs[app.StackName]
if !hasCfg || cfg == nil {
fullStatus.UnconfiguredApps = append(fullStatus.UnconfiguredApps, backup.CrossDriveSummaryItem{
StackName: app.StackName,
DisplayName: app.DisplayName,
})
continue
}
item := backup.CrossDriveSummaryItem{
StackName: app.StackName,
DisplayName: app.DisplayName,
Method: cfg.Method,
DestPath: cfg.DestinationPath,
DestLabel: destLabels[cfg.DestinationPath],
Schedule: cfg.Schedule,
LastStatus: cfg.LastStatus,
SizeHuman: cfg.LastSizeHuman,
}
switch cfg.Method {
case "rsync":
item.MethodLabel = "rsync"
case "restic":
item.MethodLabel = "restic"
default:
item.MethodLabel = cfg.Method
}
switch cfg.Schedule {
case "daily":
item.ScheduleLabel = "Naponta"
case "weekly":
item.ScheduleLabel = "Hetente"
default:
item.ScheduleLabel = "Kézi"
}
if cfg.LastRun != "" {
if t, err := time.Parse(time.RFC3339, cfg.LastRun); err == nil {
item.LastRunShort = t.In(getTimezone()).Format("01-02 15:04")
}
}
fullStatus.CrossDriveSummary = append(fullStatus.CrossDriveSummary, item)
// Destination health warning (tiered validation)
if cfg.Enabled && cfg.DestinationPath != "" {
health := system.CheckBackupDestination(cfg.DestinationPath)
if health.Warning != "" {
prefix := "⚠️"
if health.Severity == "critical" {
prefix = "🔴"
}
fullStatus.CrossDriveWarnings = append(fullStatus.CrossDriveWarnings,
fmt.Sprintf("%s %s: %s", prefix, app.DisplayName, health.Warning))
}
}
}
// Build unified per-app backup rows for the new UI
data["AppBackupRows"] = s.buildAppBackupRows(fullStatus, crossConfigs, destLabels)
// Top-level warning: no user data backed up at all
hasAnyCrossDrive := false
hasAnyHDDApp := false
for _, app := range fullStatus.AppDataInfo {
if app.HasHDDData {
hasAnyHDDApp = true
if cfg, ok := crossConfigs[app.StackName]; ok && cfg != nil && cfg.Enabled {
hasAnyCrossDrive = true
}
}
}
if hasAnyHDDApp && !hasAnyCrossDrive {
data["NoUserDataBackupWarning"] = true
}
// Build unified per-app backup rows for the app-data backup UI.
// Disk-tier (cross-drive / restic) backup has moved to the host agent.
data["AppBackupRows"] = s.buildAppBackupRows(fullStatus)
data["Backup"] = fullStatus
// Restic password for display
if pw, err := s.backupMgr.GetResticPassword(); err == nil {
data["ResticPassword"] = pw
}
// Részletek section: DB dump total size
// DB dump total size
var dbDumpTotalBytes int64
for _, f := range fullStatus.DumpFiles {
dbDumpTotalBytes += f.Size
}
data["DBDumpTotalBytes"] = dbDumpTotalBytes
// Részletek section: enrich per-drive repo stats with storage labels
for i := range fullStatus.PerDriveRepoStats {
for _, sp := range storagePaths {
if strings.HasPrefix(fullStatus.PerDriveRepoStats[i].DrivePath, sp.Path) ||
fullStatus.PerDriveRepoStats[i].DrivePath == sp.Path {
fullStatus.PerDriveRepoStats[i].DriveLabel = sp.Label
break
}
}
if fullStatus.PerDriveRepoStats[i].DriveLabel == "" {
fullStatus.PerDriveRepoStats[i].DriveLabel = filepath.Base(fullStatus.PerDriveRepoStats[i].DrivePath)
}
}
data["PerDriveRepoStats"] = fullStatus.PerDriveRepoStats
// Részletek section: group Tier 2 items by destination drive
tier2GroupMap := make(map[string]*Tier2DriveGroup)
for _, item := range fullStatus.CrossDriveSummary {
if item.DestPath == "" {
continue
}
grp, exists := tier2GroupMap[item.DestPath]
if !exists {
grp = &Tier2DriveGroup{
DestPath: item.DestPath,
DestLabel: item.DestLabel,
}
if grp.DestLabel == "" {
grp.DestLabel = filepath.Base(item.DestPath)
}
tier2GroupMap[item.DestPath] = grp
}
grp.Items = append(grp.Items, item)
}
var tier2Groups []Tier2DriveGroup
for _, grp := range tier2GroupMap {
tier2Groups = append(tier2Groups, *grp)
}
data["Tier2DriveGroups"] = tier2Groups
} else {
data["Backup"] = nil
}
@@ -752,13 +558,6 @@ func (s *Server) backupsHandler(w http.ResponseWriter, r *http.Request) {
s.executeTemplate(w, r, "backups", data)
}
// Tier2DriveGroup holds grouped Tier 2 cross-drive backup items for one destination drive.
type Tier2DriveGroup struct {
DestPath string
DestLabel string
Items []backup.CrossDriveSummaryItem
}
// AppBackupRow holds per-tier backup information for one app on the backup page.
type AppBackupRow struct {
StackName string
@@ -804,13 +603,9 @@ type AppBackupRow struct {
}
// buildAppBackupRows constructs one AppBackupRow per deployed app for the backup page.
func (s *Server) buildAppBackupRows(
status *backup.FullBackupStatus,
crossConfigs map[string]*settings.CrossDriveBackup,
destLabels map[string]string,
) []AppBackupRow {
loc := getTimezone()
// Disk-tier (cross-drive / restic) backup has moved to the host agent; this now
// reflects only the app-data backup (DB dumps + Docker-volume tars).
func (s *Server) buildAppBackupRows(status *backup.FullBackupStatus) []AppBackupRow {
// Build DB stack lookup
dbStacks := make(map[string]bool)
for _, db := range status.DiscoveredDBs {
@@ -820,17 +615,6 @@ func (s *Server) buildAppBackupRows(
dbStacks[f.StackName] = true
}
// Tier 1 timestamps (shared across all apps — single nightly job)
tier1LastRun := ""
tier1LastStatus := ""
if status.LastBackup != nil {
tier1LastRun = status.LastBackup.LastRun.In(loc).Format("01-02 15:04")
if status.LastBackup.Success {
tier1LastStatus = "ok"
} else {
tier1LastStatus = "error"
}
}
tier1DBStatus := ""
if status.LastDBDump != nil {
if status.LastDBDump.Success {
@@ -884,115 +668,18 @@ func (s *Server) buildAppBackupRows(
HasDB: hasDB,
HasVolumeData: app.HasVolumeData,
DriveDisconnected: driveDisconnected,
StorageLabel: app.StorageLabel,
HDDSizeHuman: app.HDDSizeHuman,
BackupContents: contents,
Tier1LastRun: tier1LastRun,
Tier1LastStatus: tier1LastStatus,
Tier1DBStatus: tier1DBStatus,
StorageLabel: app.StorageLabel,
HDDSizeHuman: app.HDDSizeHuman,
BackupContents: contents,
Tier1DBStatus: tier1DBStatus,
}
// Status dot — start as yellow (1 tier only)
row.Status = "yellow"
row.StatusText = "Csak helyi mentés (1 szint)"
cfg, hasCfg := crossConfigs[app.StackName]
if !hasCfg || cfg == nil || !cfg.Enabled {
// Only Tier 1 — no second copy
row.Tier2Configured = false
} else {
row.Tier2Configured = true
row.Tier2Dest = destLabels[cfg.DestinationPath]
if row.Tier2Dest == "" {
row.Tier2Dest = cfg.DestinationPath
}
switch cfg.Schedule {
case "daily":
row.Tier2Schedule = "Naponta"
case "weekly":
row.Tier2Schedule = "Hetente"
default:
row.Tier2Schedule = cfg.Schedule
}
if cfg.LastRun != "" {
if t, err := time.Parse(time.RFC3339, cfg.LastRun); err == nil {
row.Tier2LastRun = t.In(loc).Format("01-02 15:04")
}
}
row.Tier2LastStatus = cfg.LastStatus
row.Tier2LastError = cfg.LastError
row.Tier2SizeHuman = cfg.LastSizeHuman
switch cfg.LastStatus {
case "ok":
row.Tier2StatusBadge = "Sikeres"
row.Status = "green"
row.StatusText = "Mentés rendben"
case "error":
row.Tier2StatusBadge = "Hiba"
// Status stays yellow
row.StatusText = "Utolsó mentés sikertelen"
case "running":
row.Tier2StatusBadge = "Fut..."
default:
row.Tier2StatusBadge = "—"
// Tier2 configured but never run — stay yellow
}
// Check if Tier2 destination drive is disconnected
if cfg.DestinationPath != "" {
for dp := range disconnectedPaths {
if cfg.DestinationPath == dp || strings.HasPrefix(cfg.DestinationPath, dp+"/") {
row.Tier2DestDisconnected = true
break
}
}
}
// Also treat as disconnected if dest was removed from storage entirely
if cfg.DestinationPath != "" && !row.Tier2DestDisconnected {
if !s.settings.IsStoragePathKnown(cfg.DestinationPath) {
row.Tier2DestDisconnected = true
}
}
// Check if Tier2 destination drive is inactive (not schedulable)
if cfg.DestinationPath != "" && !row.Tier2DestDisconnected {
if !s.settings.IsStoragePathSchedulable(cfg.DestinationPath) {
row.Tier2DestInactive = true
}
}
if row.Tier2DestDisconnected {
// Disconnected destination — treat as paused, not failed
row.Status = "yellow"
row.StatusText = "2. mentés szünetel — cél meghajtó leválasztva"
} else if row.Tier2DestInactive {
// Inactive destination — treat as paused
row.Status = "yellow"
row.StatusText = "2. mentés szünetel — cél meghajtó inaktív"
} else if cfg.DestinationPath != "" && s.crossDriveRunner != nil {
// Destination health check — can downgrade green to yellow/red
if err := s.crossDriveRunner.ValidateDestination(cfg.DestinationPath); err != nil {
if strings.Contains(err.Error(), "does not exist") || strings.Contains(err.Error(), "not writable") {
row.Status = "red"
row.StatusText = "Mentési cél nem elérhető"
} else if row.Status != "red" {
row.Status = "yellow"
row.StatusText = "Figyelmeztetés"
}
row.Warnings = append(row.Warnings, err.Error())
}
}
}
// DB dump failure warning (affects Tier 1 quality)
// Status dot — app-data backup status
row.Status = "green"
row.StatusText = "Alkalmazás-adat mentés rendben"
if hasDB && tier1DBStatus == "error" {
if row.Status != "red" {
row.Status = "yellow"
row.StatusText = "Adatbázis mentés sikertelen"
}
row.Status = "yellow"
row.StatusText = "Adatbázis mentés sikertelen"
}
rows = append(rows, row)
@@ -1000,79 +687,6 @@ func (s *Server) buildAppBackupRows(
return rows
}
// settingsCrossBackupHandler handles POST /settings/cross-backup/{name}
// Saves or updates the cross-drive backup configuration for an app.
func (s *Server) settingsCrossBackupHandler(w http.ResponseWriter, r *http.Request, name string) {
_ = r.ParseForm()
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] settingsCrossBackupHandler: stack=%s from %s", name, r.RemoteAddr)
}
enabled := r.FormValue("cross_drive_enabled") == "on"
// Preserve existing runtime status fields and config when disabling
existing := s.settings.GetCrossDriveConfig(name)
var destPath, schedule string
if enabled {
destPath = r.FormValue("cross_drive_dest")
schedule = r.FormValue("cross_drive_schedule")
if schedule != "daily" && schedule != "weekly" {
schedule = "daily"
}
} else if existing != nil {
// Preserve existing settings when disabling
destPath = existing.DestinationPath
schedule = existing.Schedule
}
// Validate destination path against registered storage paths (H11 fix — matches API handler).
if enabled && destPath != "" {
registeredPaths := s.settings.GetStoragePaths()
validDest := false
for _, sp := range registeredPaths {
if destPath == sp.Path {
validDest = true
break
}
}
if !validDest {
s.logger.Printf("[WARN] [web] Cross-drive backup: rejected invalid dest path %q for %s", destPath, name)
http.Redirect(w, r, "/stacks/"+name+"/deploy?flash_error="+url.QueryEscape("Érvénytelen célútvonal: "+destPath), http.StatusFound)
return
}
}
var cfg *settings.CrossDriveBackup
if destPath != "" || existing != nil {
cfg = &settings.CrossDriveBackup{
Enabled: enabled,
Method: "rsync",
DestinationPath: destPath,
Schedule: schedule,
}
if existing != nil {
cfg.LastRun = existing.LastRun
cfg.LastStatus = existing.LastStatus
cfg.LastError = existing.LastError
cfg.LastDuration = existing.LastDuration
cfg.LastSizeHuman = existing.LastSizeHuman
}
}
if err := s.settings.SetCrossDriveConfig(name, cfg); err != nil {
s.logger.Printf("[ERROR] [web] Failed to save cross-drive config for %s: %v", name, err)
http.Redirect(w, r, "/stacks/"+name+"/deploy?flash_error=Hiba+a+ment%C3%A9si+be%C3%A1ll%C3%ADt%C3%A1s+ment%C3%A9sakor", http.StatusFound)
return
}
s.logger.Printf("[INFO] [web] Cross-drive backup config saved for %s: dest=%s schedule=%s enabled=%v",
name, destPath, schedule, enabled)
http.Redirect(w, r, "/stacks/"+name+"/deploy?flash=Ment%C3%A9si+be%C3%A1ll%C3%ADt%C3%A1s+mentve.", http.StatusFound)
}
func (s *Server) backupRestoreHandler(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
@@ -1096,12 +710,7 @@ func (s *Server) backupRestoreHandler(w http.ResponseWriter, r *http.Request) {
s.logger.Printf("[WARN] [web] Restore requested: stack=%s, snapshot=%s from %s", stackName, snapshotID, r.RemoteAddr)
start := time.Now()
var err error
if snapshotID == "tier2-rsync" {
err = s.backupMgr.RestoreAppFromTier2(stackName)
} else {
err = s.backupMgr.RestoreApp(stackName, snapshotID)
}
err := s.backupMgr.RestoreApp(stackName, snapshotID)
if err != nil {
s.logger.Printf("[ERROR] [web] Restore failed: %v", err)
if s.isDebug() {
controller/internal/web/server.go
+30 -120
View File
@@ -17,31 +17,28 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/integrations"
"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
"gitea.dooplex.hu/admin/felhom-controller/internal/selfupdate"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
"gitea.dooplex.hu/admin/felhom-controller/internal/storage"
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
)
type Server struct {
cfg *config.Config
stackMgr *stacks.Manager
cpuCollector *system.CPUCollector
backupMgr *backup.Manager
crossDriveRunner *backup.CrossDriveRunner
scheduler *scheduler.Scheduler
settings *settings.Settings
alertManager *AlertManager
notifier *notify.Notifier
updater *selfupdate.Updater
logger *log.Logger
version string
encKey []byte // AES-256 key for decrypting app.yaml values
tmpl *template.Template
cfg *config.Config
stackMgr *stacks.Manager
cpuCollector *system.CPUCollector
backupMgr *backup.Manager
scheduler *scheduler.Scheduler
settings *settings.Settings
alertManager *AlertManager
notifier *notify.Notifier
updater *selfupdate.Updater
logger *log.Logger
version string
encKey []byte // AES-256 key for decrypting app.yaml values
tmpl *template.Template
sessions map[string]*session
sessionsMu sync.RWMutex
@@ -50,26 +47,9 @@ type Server struct {
done chan struct{}
closeOnce sync.Once
// Disk operation state (format/migrate jobs)
diskJobMu sync.Mutex
diskJob *activeDiskJob
// Active raw mount for the attach wizard (empty when not in use)
activeRawMount string
// Guard for FileBrowser sync — prevents concurrent file writes (H5 fix)
fileBrowserMu sync.Mutex
// Drive migration
driveMigrator *storage.DriveMigrator
// DR restore mode state
restoreMu sync.RWMutex
restorePlan *backup.RestorePlan
// Storage watchdog (set after construction to break init ordering)
storageWatchdog *monitor.StorageWatchdog
// Hub push status callback — set via SetHubPushStatus for monitoring page
hubPushStatusFn func() HubPushStatusData
@@ -88,29 +68,28 @@ type Server struct {
startTime time.Time
}
func NewServer(cfg *config.Config, stackMgr *stacks.Manager, cpuCollector *system.CPUCollector, backupMgr *backup.Manager, crossDrive *backup.CrossDriveRunner, sched *scheduler.Scheduler, sett *settings.Settings, alertMgr *AlertManager, notif *notify.Notifier, updater *selfupdate.Updater, logger *log.Logger, version string) *Server {
func NewServer(cfg *config.Config, stackMgr *stacks.Manager, cpuCollector *system.CPUCollector, backupMgr *backup.Manager, sched *scheduler.Scheduler, sett *settings.Settings, alertMgr *AlertManager, notif *notify.Notifier, updater *selfupdate.Updater, logger *log.Logger, version string) *Server {
s := &Server{
cfg: cfg,
stackMgr: stackMgr,
cpuCollector: cpuCollector,
backupMgr: backupMgr,
crossDriveRunner: crossDrive,
scheduler: sched,
settings: sett,
alertManager: alertMgr,
notifier: notif,
updater: updater,
logger: logger,
version: version,
sessions: make(map[string]*session),
loginAttempts: make(map[string]*loginAttempt),
done: make(chan struct{}),
cfg: cfg,
stackMgr: stackMgr,
cpuCollector: cpuCollector,
backupMgr: backupMgr,
scheduler: sched,
settings: sett,
alertManager: alertMgr,
notifier: notif,
updater: updater,
logger: logger,
version: version,
sessions: make(map[string]*session),
loginAttempts: make(map[string]*loginAttempt),
done: make(chan struct{}),
}
if cfg.Logging.Level == "debug" {
logger.Printf("[DEBUG] [web] NewServer: initializing web server v%s", version)
logger.Printf("[DEBUG] [web] NewServer: backup=%v crossDrive=%v scheduler=%v alertMgr=%v notifier=%v updater=%v",
backupMgr != nil, crossDrive != nil, sched != nil, alertMgr != nil, notif != nil, updater != nil)
logger.Printf("[DEBUG] [web] NewServer: backup=%v scheduler=%v alertMgr=%v notifier=%v updater=%v",
backupMgr != nil, sched != nil, alertMgr != nil, notif != nil, updater != nil)
}
s.loadTemplates()
@@ -155,23 +134,6 @@ func (s *Server) loadTemplates() {
}
}
// SetRestoreState puts the server into DR restore mode with the given plan.
func (s *Server) SetRestoreState(plan *backup.RestorePlan) {
s.restoreMu.Lock()
defer s.restoreMu.Unlock()
s.restorePlan = plan
}
// SetStorageWatchdog sets the storage watchdog for disconnect/reconnect operations.
func (s *Server) SetStorageWatchdog(w *monitor.StorageWatchdog) {
s.storageWatchdog = w
}
// SetDriveMigrator sets the drive migration engine for full drive migration.
func (s *Server) SetDriveMigrator(dm *storage.DriveMigrator) {
s.driveMigrator = dm
}
// HubPushStatusData holds hub push status for the monitoring page.
type HubPushStatusData struct {
LastAttempt time.Time
@@ -230,13 +192,6 @@ func (s *Server) ServeDebugAPI(w http.ResponseWriter, r *http.Request) {
s.handleDebugAPI(w, r)
}
// InRestoreMode returns true if the server is in DR restore mode.
func (s *Server) InRestoreMode() bool {
s.restoreMu.RLock()
defer s.restoreMu.RUnlock()
return s.restorePlan != nil
}
// ServeHTTP handles all non-API web requests.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
@@ -245,30 +200,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.logger.Printf("[DEBUG] [web] ServeHTTP: %s %s from %s", r.Method, path, r.RemoteAddr)
}
// DR restore mode: intercept all routes except restore page, static, and restore API
if s.InRestoreMode() {
switch {
case path == "/restore":
s.restorePageHandler(w, r)
return
case path == "/api/restore/status":
s.apiRestoreStatus(w, r)
return
case path == "/api/restore/all" && r.Method == http.MethodPost:
s.apiRestoreAll(w, r)
return
case path == "/api/restore/skip" && r.Method == http.MethodPost:
s.apiRestoreSkip(w, r)
return
case strings.HasPrefix(path, "/static/"):
// Allow static assets through
default:
// Redirect everything else to the restore page
http.Redirect(w, r, "/restore", http.StatusFound)
return
}
}
switch {
case path == "/" || path == "/dashboard":
s.dashboardHandler(w, r)
@@ -296,25 +227,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.settingsStorageSchedulableHandler(w, r)
case path == "/settings/storage/label" && r.Method == http.MethodPost:
s.settingsStorageLabelHandler(w, r)
case strings.HasPrefix(path, "/settings/cross-backup/") && r.Method == http.MethodPost:
name := strings.TrimPrefix(path, "/settings/cross-backup/")
s.settingsCrossBackupHandler(w, r, name)
case path == "/backup/restore" && r.Method == http.MethodPost:
s.backupRestoreHandler(w, r)
case path == "/settings/storage/init":
s.storageInitHandler(w, r)
case path == "/settings/storage/attach":
s.storageAttachHandler(w, r)
case path == "/settings/storage/migrate-drive":
s.migrateDrivePageHandler(w, r)
case strings.HasPrefix(path, "/stacks/") && strings.HasSuffix(path, "/export"):
name := strings.TrimPrefix(path, "/stacks/")
name = strings.TrimSuffix(name, "/export")
s.exportPageHandler(w, r, name)
case strings.HasPrefix(path, "/stacks/") && strings.HasSuffix(path, "/migrate"):
name := strings.TrimPrefix(path, "/stacks/")
name = strings.TrimSuffix(name, "/migrate")
s.migratePageHandler(w, r, name)
case strings.HasPrefix(path, "/stacks/") && strings.HasSuffix(path, "/logs"):
name := strings.TrimPrefix(path, "/stacks/")
name = strings.TrimSuffix(name, "/logs")
@@ -440,14 +358,6 @@ func (s *Server) findStackBySubdomain(subdomain string) (*stacks.Stack, bool) {
return nil, false
}
// ServeStorageAPI handles /api/storage/* routes (JSON API for disk operations).
func (s *Server) ServeStorageAPI(w http.ResponseWriter, r *http.Request) {
if s.isDebug() {
s.logger.Printf("[DEBUG] [web] ServeStorageAPI: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
}
s.storageAPIHandler(w, r)
}
// primaryHDDPath returns the default storage path, or the legacy config value.
func (s *Server) primaryHDDPath() string {
if p := s.settings.GetDefaultStoragePath(); p != "" {
controller/internal/web/storage_handlers.go
-1600
View File
File diff suppressed because it is too large Load Diff