From a4de90def3f70540fc0a06d247e9a333c990c2ad Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 8 Jun 2026 11:01:39 +0200 Subject: [PATCH] refactor: extract app-data-backup into internal/appbackup (no behaviour change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the stateless, keep-side app-data backup primitives out of internal/backup/ into a new self-contained internal/appbackup/ package: - dbdump.go: DB dump discovery/execution (DiscoverDatabases, DumpOne, ...) - appdata.go: StackDataProvider + app-data/volume discovery, HumanizeBytes - paths.go: keep-side path helpers (AppDBDumpPath, AppVolumeDumpPath, AppDataDir) backup/ keeps every name available via type/const aliases + one-line function forwarders (appbackup_bridge.go), so the still-present delete-side code (restic, cross-drive, drive-mount) and the both-side consumers (web/api/report) compile unchanged. The keep-only consumers appexport and storage are rewired to import appbackup directly and no longer import backup. This is the Part-2 prerequisite for the Proxmox port: appbackup has zero references to restic/cross-drive/drive-mount and does not import backup, so the delete-side can later be removed without breaking app-data backup or appexport. Behaviour-preserving: pure move + import/qualifier rewrites, no logic edits. The four Manager methods (RunDBDumps/DumpAppVolumes/DumpAppVolumesSafe share the delete-side mutex/status state; RestoreAppFromTier2 reads the cross-drive mirror) intentionally stay on Manager and delegate to appbackup — for the re-platform step. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 9 ++ controller/README.md | 5 +- .../internal/{backup => appbackup}/appdata.go | 12 +- .../internal/{backup => appbackup}/dbdump.go | 2 +- controller/internal/appbackup/paths.go | 31 ++++++ controller/internal/appexport/export.go | 8 +- .../internal/backup/appbackup_bridge.go | 105 ++++++++++++++++++ controller/internal/backup/paths.go | 25 +---- controller/internal/storage/migrate.go | 6 +- controller/internal/storage/migrate_drive.go | 8 +- 10 files changed, 172 insertions(+), 39 deletions(-) rename controller/internal/{backup => appbackup}/appdata.go (93%) rename controller/internal/{backup => appbackup}/dbdump.go (99%) create mode 100644 controller/internal/appbackup/paths.go create mode 100644 controller/internal/backup/appbackup_bridge.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 16ef755..43011ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## Changelog +### Refactor — extract app-data-backup primitives into `internal/appbackup` (no behaviour change) (2026-06-08) + +#### Changed +- **New package `internal/appbackup/`**: extracted the stateless, keep-side app-data backup primitives out of `internal/backup/` — DB dump discovery/execution (`dbdump.go`: `DiscoverDatabases`, `DumpAll`, `DumpOne`, `ValidateDump`, `ListDumpFiles`), Docker-volume/app-data discovery (`appdata.go`: `StackDataProvider`, `DiscoverAppData`, `ParseComposeNamedVolumes`, `ResolveDockerVolumeNames`, `HumanizeBytes`), and keep-side path helpers (`paths.go`: `FelhomDataDir`, `PrimaryBackupPath`, `AppDBDumpPath`, `AppVolumeDumpPath`, `AppDataDir`). Pure move — logic unchanged. +- **backup/appbackup_bridge.go** (new): re-exposes the moved symbols to the `backup` package via type/const aliases and one-line function forwarders, so the still-present disk/host-side code (restic, cross-drive, drive-mount) and the both-side consumers (web, api, report) compile unchanged. +- **appexport/export.go, storage/migrate.go, storage/migrate_drive.go**: rewired to import `internal/appbackup` directly and dropped their `internal/backup` import — these keep-side consumers are now independent of the delete-side code. +- **Why**: Part-2 prerequisite for the Proxmox port. Isolating the keep-side now (as a separate green, behaviour-identical commit) means the disk/host-side code can later be removed without breaking app-data backup or `appexport`. `appbackup` has zero references to restic/cross-drive/drive-mount and does not import `backup` (no import cycle). +- **Not moved (documented coupling)**: the `*Manager` methods `RunDBDumps`/`DumpAppVolumes`/`DumpAppVolumesSafe` (share one mutex/running-flag + status state with the delete-side `RunBackup`) and `RestoreAppFromTier2` (intrinsically reads the cross-drive mirror via `copyFile`/`AppSecondaryRsyncPath`) stay on `Manager`; they delegate to `appbackup` and are left for the later re-platform step. + ### v0.34.0 — Backup safety: stop-before-dump, streaming restore, health check, per-app restic, infra configs (2026-02-28) #### Changed diff --git a/controller/README.md b/controller/README.md index 79321f9..40f8bcc 100644 --- a/controller/README.md +++ b/controller/README.md @@ -81,7 +81,7 @@ A single, lightweight Go container that replaces Portainer + scattered systemd s - **Pure Go, no frameworks** — stdlib `net/http` + `html/template`. Only external deps: `bcrypt`, `yaml.v3`, `modernc.org/sqlite` (pure Go, no CGO). - **Privileged container** — Required for disk operations (format, mount, fstab), `/dev` access, and Docker socket control. - **`/host-dev` indirection** — Docker overrides `/dev` with a tmpfs. The host's `/dev` is mounted at `/host-dev` to access block devices. -- **`StackDataProvider` interface** — Breaks circular import between backup and stacks packages. Implemented by `stackAdapter` in `main.go`. Provides `GetStackHDDPath()` for per-drive backup routing. +- **`StackDataProvider` interface** — Breaks circular import between the backup packages and stacks. Defined in `internal/appbackup` (and re-exposed via a type alias in `internal/backup`). Implemented by `stackAdapter` in `main.go`. Provides `GetStackHDDPath()` for per-drive backup routing. - **Atomic file writes** — All persistent state (`settings.json`, `app.yaml`) written to `.tmp` then `os.Rename` for crash safety. - **`go:embed` templates** — All HTML/CSS/JS compiled into the binary. No runtime file dependencies. - **Europe/Budapest timezone** — All scheduled jobs, timestamps, and UI labels use Hungarian timezone. @@ -95,7 +95,8 @@ A single, lightweight Go container that replaces Portainer + scattered systemd s | **Stacks** | `internal/stacks/` | Compose operations, scanning, `.felhom.yml` metadata, deploy/delete flow | | **Crypto** | `internal/crypto/` | AES-256-GCM encryption for sensitive app.yaml values (passwords, secrets), key management | | **Sync** | `internal/sync/` | Git-based app catalog sync (clone/pull, content-hash copy) | -| **Backup** | `internal/backup/` | Per-drive 3-layer backup: DB dumps → restic snapshots → cross-drive copies, restore | +| **AppBackup** | `internal/appbackup/` | Self-contained app-data backup primitives: DB dump discovery/execution (`DiscoverDatabases`, `DumpOne`), Docker-volume/app-data discovery (`StackDataProvider`, `DiscoverAppData`), keep-side path helpers (`AppDBDumpPath`, `AppVolumeDumpPath`, `AppDataDir`). No dependency on restic/cross-drive/drive-mount. Imported directly by `appexport` and `storage`. | +| **Backup** | `internal/backup/` | Per-drive 3-layer backup: DB dumps → restic snapshots → cross-drive copies, restore. Re-exposes the `appbackup` primitives via aliases/forwarders (`appbackup_bridge.go`) for the disk/host-side code and the web/api/report consumers. | | **Storage** | `internal/storage/` | Disk scanning (`lsblk`), partitioning (`sfdisk`), formatting (`mkfs.ext4`), mounting, data migration (`rsync`) | | **System** | `internal/system/` | System info (`/proc`), CPU collector, mount points, disk usage, FS info | | **Monitor** | `internal/monitor/` | System health checks, storage watchdog, legacy Healthchecks pinger (deprecated) | diff --git a/controller/internal/backup/appdata.go b/controller/internal/appbackup/appdata.go similarity index 93% rename from controller/internal/backup/appdata.go rename to controller/internal/appbackup/appdata.go index 24bb590..f446fd4 100644 --- a/controller/internal/backup/appdata.go +++ b/controller/internal/appbackup/appdata.go @@ -1,4 +1,4 @@ -package backup +package appbackup import ( "context" @@ -13,12 +13,12 @@ import ( "gopkg.in/yaml.v3" ) -// StackDataProvider provides stack data to the backup package without circular imports. +// StackDataProvider provides stack data to the backup packages without circular imports. type StackDataProvider interface { GetStackComposePath(name string) (composePath string, ok bool) ListDeployedStacks() []StackSummary GetStackHDDMounts(name string) []string - GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD) + GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD) GetDockerVolumes(name string) []string // full Docker volume names (project-prefixed) StopStack(name string) error StartStack(name string) error @@ -179,6 +179,12 @@ func appDirSize(path string) (int64, string) { return size, humanizeBytes(size) } +// HumanizeBytes converts bytes to a human-readable string. +// Exported so the backup package can forward to it (shared helper). +func HumanizeBytes(b int64) string { + return humanizeBytes(b) +} + // humanizeBytes converts bytes to a human-readable string. func humanizeBytes(b int64) string { const ( diff --git a/controller/internal/backup/dbdump.go b/controller/internal/appbackup/dbdump.go similarity index 99% rename from controller/internal/backup/dbdump.go rename to controller/internal/appbackup/dbdump.go index 1252036..b96eed2 100644 --- a/controller/internal/backup/dbdump.go +++ b/controller/internal/appbackup/dbdump.go @@ -1,4 +1,4 @@ -package backup +package appbackup import ( "bufio" diff --git a/controller/internal/appbackup/paths.go b/controller/internal/appbackup/paths.go new file mode 100644 index 0000000..a156fc3 --- /dev/null +++ b/controller/internal/appbackup/paths.go @@ -0,0 +1,31 @@ +// Package appbackup holds the self-contained app-data backup primitives: +// database dump and Docker-volume archive discovery/execution, plus the +// keep-side storage path helpers. It depends only on stable abstractions +// (the StackDataProvider interface) and has no dependency on the restic, +// cross-drive, or drive-mount code in the backup package. +package appbackup + +import "path/filepath" + +// FelhomDataDir is the namespace directory on storage drives for all felhom-managed data. +const FelhomDataDir = "felhom-data" + +// PrimaryBackupPath returns the root primary backup directory for a drive. +func PrimaryBackupPath(drivePath string) string { + return filepath.Join(drivePath, FelhomDataDir, "backups", "primary") +} + +// AppDBDumpPath returns the DB dump directory for an app on its home drive. +func AppDBDumpPath(drivePath, stackName string) string { + return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", stackName, "db-dumps") +} + +// AppVolumeDumpPath returns the directory for Docker volume dump tars on an app's home drive. +func AppVolumeDumpPath(drivePath, stackName string) string { + return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", stackName, "volume-dumps") +} + +// AppDataDir returns the app data directory path on a drive. +func AppDataDir(drivePath, stackName string) string { + return filepath.Join(drivePath, FelhomDataDir, "appdata", stackName) +} diff --git a/controller/internal/appexport/export.go b/controller/internal/appexport/export.go index 6b5d867..481f011 100644 --- a/controller/internal/appexport/export.go +++ b/controller/internal/appexport/export.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "gitea.dooplex.hu/admin/felhom-controller/internal/backup" + "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" ) // Step tracks one step of an export/import operation. @@ -510,7 +510,7 @@ func (e *Exporter) dumpDatabase(stackName, dbDir string, manifest *Manifest) boo defer cancel() e.debugf("discovering databases (looking for stack %s)...", stackName) - dbs, err := backup.DiscoverDatabases(ctx, e.logger, e.debug) + dbs, err := appbackup.DiscoverDatabases(ctx, e.logger, e.debug) if err != nil { e.logger.Printf("[WARN] Export: DB discovery error: %v", err) return false @@ -520,7 +520,7 @@ func (e *Exporter) dumpDatabase(stackName, dbDir string, manifest *Manifest) boo e.debugf(" db[%d]: stack=%s container=%s type=%s", i, dbs[i].StackName, dbs[i].ContainerName, dbs[i].DBType) } - var stackDB *backup.DiscoveredDB + var stackDB *appbackup.DiscoveredDB for i := range dbs { if dbs[i].StackName == stackName { stackDB = &dbs[i] @@ -534,7 +534,7 @@ func (e *Exporter) dumpDatabase(stackName, dbDir string, manifest *Manifest) boo e.debugf("matched DB: container=%s type=%s", stackDB.ContainerName, stackDB.DBType) dumpStart := time.Now() - result := backup.DumpOne(ctx, *stackDB, dbDir, e.logger, e.debug) + result := appbackup.DumpOne(ctx, *stackDB, dbDir, e.logger, e.debug) if result.Error != nil { e.logger.Printf("[WARN] Export: DB dump failed for %s: %v", stackName, result.Error) return false diff --git a/controller/internal/backup/appbackup_bridge.go b/controller/internal/backup/appbackup_bridge.go new file mode 100644 index 0000000..6aaf796 --- /dev/null +++ b/controller/internal/backup/appbackup_bridge.go @@ -0,0 +1,105 @@ +package backup + +// This file bridges the backup package to internal/appbackup, where the +// self-contained app-data backup primitives (DB dump, Docker-volume archive +// discovery, keep-side path helpers) now live. The backup package keeps these +// names available — via type/const aliases and thin function forwarders — so +// the (still present) delete-side code and the both-side consumers (web, api, +// report) compile unchanged. Behaviour is identical: the forwarders call +// straight through to appbackup. +// +// Go has no function aliasing, so the functions are one-line forwarders while +// the types/consts use real aliases. + +import ( + "context" + "log" + + "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" +) + +// --- type aliases (appdata) --- + +type StackDataProvider = appbackup.StackDataProvider +type StackSummary = appbackup.StackSummary +type AppBackupInfo = appbackup.AppBackupInfo +type AppDataPath = appbackup.AppDataPath +type AppDockerVolume = appbackup.AppDockerVolume + +// --- type aliases (dbdump) --- + +type DBType = appbackup.DBType +type DiscoveredDB = appbackup.DiscoveredDB +type DumpResult = appbackup.DumpResult +type DumpValidation = appbackup.DumpValidation +type DumpFileInfo = appbackup.DumpFileInfo + +// --- const aliases --- + +const ( + DBTypePostgres = appbackup.DBTypePostgres + DBTypeMariaDB = appbackup.DBTypeMariaDB +) + +// FelhomDataDir is the namespace directory on storage drives for all felhom-managed data. +const FelhomDataDir = appbackup.FelhomDataDir + +// --- function forwarders (dbdump) --- + +func DiscoverDatabases(ctx context.Context, logger *log.Logger, debug bool) ([]DiscoveredDB, error) { + return appbackup.DiscoverDatabases(ctx, logger, debug) +} + +func DumpAll(ctx context.Context, dbs []DiscoveredDB, dumpDir string, logger *log.Logger, debug bool) []DumpResult { + return appbackup.DumpAll(ctx, dbs, dumpDir, logger, debug) +} + +func DumpOne(ctx context.Context, db DiscoveredDB, dumpDir string, logger *log.Logger, debug bool) DumpResult { + return appbackup.DumpOne(ctx, db, dumpDir, logger, debug) +} + +func ValidateDump(filePath string, dbType DBType) DumpValidation { + return appbackup.ValidateDump(filePath, dbType) +} + +func ListDumpFiles(dumpDir string) ([]DumpFileInfo, error) { + return appbackup.ListDumpFiles(dumpDir) +} + +// --- function forwarders (appdata) --- + +func DiscoverAppData(provider StackDataProvider, discoveredDBs []DiscoveredDB) []AppBackupInfo { + return appbackup.DiscoverAppData(provider, discoveredDBs) +} + +func ParseComposeNamedVolumes(composePath string) []AppDockerVolume { + return appbackup.ParseComposeNamedVolumes(composePath) +} + +func ResolveDockerVolumeNames(composePath string) []string { + return appbackup.ResolveDockerVolumeNames(composePath) +} + +// humanizeBytes forwards to appbackup.HumanizeBytes; kept unexported so the +// many in-package call sites (backup.go, crossdrive.go, restore code) need no edit. +func humanizeBytes(b int64) string { + return appbackup.HumanizeBytes(b) +} + +// --- function forwarders (paths) --- + +func PrimaryBackupPath(drivePath string) string { + return appbackup.PrimaryBackupPath(drivePath) +} + +func AppDBDumpPath(drivePath, stackName string) string { + return appbackup.AppDBDumpPath(drivePath, stackName) +} + +func AppVolumeDumpPath(drivePath, stackName string) string { + return appbackup.AppVolumeDumpPath(drivePath, stackName) +} + +func AppDataDir(drivePath, stackName string) string { + return appbackup.AppDataDir(drivePath, stackName) +} diff --git a/controller/internal/backup/paths.go b/controller/internal/backup/paths.go index d0a0235..a7ff5a6 100644 --- a/controller/internal/backup/paths.go +++ b/controller/internal/backup/paths.go @@ -2,29 +2,15 @@ package backup import "path/filepath" -// FelhomDataDir is the namespace directory on storage drives for all felhom-managed data. -const FelhomDataDir = "felhom-data" - -// PrimaryBackupPath returns the root primary backup directory for a drive. -func PrimaryBackupPath(drivePath string) string { - return filepath.Join(drivePath, FelhomDataDir, "backups", "primary") -} +// Keep-side path helpers (FelhomDataDir, PrimaryBackupPath, AppDBDumpPath, +// AppVolumeDumpPath, AppDataDir) now live in internal/appbackup and are +// re-exposed here via aliases/forwarders in appbackup_bridge.go. // PrimaryResticRepoPath returns the restic repo path on a drive's primary backup. func PrimaryResticRepoPath(drivePath string) string { return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", "restic") } -// AppDBDumpPath returns the DB dump directory for an app on its home drive. -func AppDBDumpPath(drivePath, stackName string) string { - return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", stackName, "db-dumps") -} - -// AppVolumeDumpPath returns the directory for Docker volume dump tars on an app's home drive. -func AppVolumeDumpPath(drivePath, stackName string) string { - return filepath.Join(drivePath, FelhomDataDir, "backups", "primary", stackName, "volume-dumps") -} - // SecondaryBackupPath returns the root secondary backup directory for a drive. func SecondaryBackupPath(drivePath string) string { return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary") @@ -45,11 +31,6 @@ func SecondaryInfraPath(drivePath string) string { return filepath.Join(drivePath, FelhomDataDir, "backups", "secondary", "_infra") } -// AppDataDir returns the app data directory path on a drive. -func AppDataDir(drivePath, stackName string) string { - return filepath.Join(drivePath, FelhomDataDir, "appdata", stackName) -} - // InfraBackupDir returns the hidden infra backup directory on a drive. func InfraBackupDir(mountPath string) string { return filepath.Join(mountPath, ".felhom-infra-backup") diff --git a/controller/internal/storage/migrate.go b/controller/internal/storage/migrate.go index d0701bb..8a4f1d9 100644 --- a/controller/internal/storage/migrate.go +++ b/controller/internal/storage/migrate.go @@ -13,7 +13,7 @@ import ( "sync" "time" - "gitea.dooplex.hu/admin/felhom-controller/internal/backup" + "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) @@ -404,8 +404,8 @@ func (o *MigrateOrchestrator) RunEnhancedMigration( // --- Post-migration steps (all non-fatal) --- // 1. Copy DB dumps from source to destination - srcDBDumps := backup.AppDBDumpPath(req.CurrentHDDPath, req.StackName) - dstDBDumps := backup.AppDBDumpPath(req.TargetPath, req.StackName) + srcDBDumps := appbackup.AppDBDumpPath(req.CurrentHDDPath, req.StackName) + dstDBDumps := appbackup.AppDBDumpPath(req.TargetPath, req.StackName) if _, err := os.Stat(srcDBDumps); err == nil { if err := os.MkdirAll(filepath.Dir(dstDBDumps), 0755); err != nil { o.Logger.Printf("[WARN] [storage] Migration %s: failed to create DB dump dir: %v", req.StackName, err) diff --git a/controller/internal/storage/migrate_drive.go b/controller/internal/storage/migrate_drive.go index 12d680b..e9051eb 100644 --- a/controller/internal/storage/migrate_drive.go +++ b/controller/internal/storage/migrate_drive.go @@ -12,7 +12,7 @@ import ( "sync" "time" - "gitea.dooplex.hu/admin/felhom-controller/internal/backup" + "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) @@ -201,7 +201,7 @@ func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateReque // Check for conflicts on destination for _, app := range appsToMigrate { - destAppData := backup.AppDataDir(req.DestPath, app.Name) + destAppData := appbackup.AppDataDir(req.DestPath, app.Name) if info, err := os.Stat(destAppData); err == nil && info.IsDir() { entries, _ := os.ReadDir(destAppData) if len(entries) > 0 { @@ -221,7 +221,7 @@ func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateReque continue } entryPath := filepath.Join(req.SourcePath, entry.Name()) - if entry.Name() == backup.FelhomDataDir { + if entry.Name() == appbackup.FelhomDataDir { // Scan inside namespace dir, excluding restic repos from estimate subEntries, _ := os.ReadDir(entryPath) for _, sub := range subEntries { @@ -363,7 +363,7 @@ func (dm *DriveMigrator) MigrateDrive(ctx context.Context, req DriveMigrateReque send("verifying", "Másolat ellenőrzése...", 62) for _, app := range appsToMigrate { - destAppData := backup.AppDataDir(req.DestPath, app.Name) + destAppData := appbackup.AppDataDir(req.DestPath, app.Name) if _, err := os.Stat(destAppData); os.IsNotExist(err) { // appdata might not exist for all apps (SSD-only apps that share the drive) // Only warn, don't fail