admin/felhom-controller
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

slice 8C Phase B.2 + C.1/C.2: retire disk subsystem + rewire disk mgmt to agent

Browse Source 
Retired (~12.3k LOC): internal/storage/* (scan/format/attach/migrate/safety),
backup restic/crossdrive/restore_drives/disk_layout/local_infra/restore_scan/
paths + restore_app, report/infra_backup*/infra_pull, setup/scanner,
monitor/watchdog+pinger, web/storage_handlers+handler_restore. Surgically split
backup.Manager to app-data only (DB dumps + volume tars + app restore; dropped
restic + cross-drive + snapshot history). Fixed router/main/web wiring.
Added agent-backed disk API (web/agent_disk_handlers.go): /api/disks list/
assign/eject/format proxying agentapi; data-bearing format refusal -> HTTP 409
'operator authorization required'. report/config_pull.go keeps the setup
fresh-install config download. go build + go test green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
authentik Default Admin
2026-06-10 13:57:27 +02:00
parent 0294513906
commit abe4e8e619
47 changed files with 404 additions and 12317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controller/internal/backup/restore.go
+16 -295
View File
@@ -5,27 +5,24 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
// snapshotIDRe validates restic snapshot IDs: 8-64 lowercase hex characters.
var snapshotIDRe = regexp.MustCompile(`^[0-9a-f]{8,64}$`)
// RestoreApp restores an app from a restic snapshot.
// All apps get config + DB dump restored. Apps with HDD data also get user data restored.
// RestoreApp restores an app's data from its on-disk app-data backup.
//
// Disk-tier (restic snapshot) restore has moved to the host agent. This keep-side
// restore re-imports the Docker-volume tar dumps that the app-data backup produced
// (AppVolumeDumpPath) and relies on the DB dumps already present on the app's drive.
// The stack is stopped before the volume import and restarted after.
//
// snapshotID is retained for API/UI signature compatibility; with restic removed it
// is only used for logging (the source of truth is now the on-disk volume tars).
func (m *Manager) RestoreApp(stackName, snapshotID string) error {
if m.stackProvider == nil {
return fmt.Errorf("stack provider not configured")
}
// Validate snapshot ID format
if !snapshotIDRe.MatchString(snapshotID) {
return fmt.Errorf("invalid snapshot ID: must be 8-64 lowercase hex characters")
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: stack=%s, snapshotID=%s", stackName, snapshotID)
}
@@ -44,87 +41,24 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
m.mu.Unlock()
}()
// Determine what to restore
hddMounts := m.stackProvider.GetStackHDDMounts(stackName)
hasHDD := len(hddMounts) > 0
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: %s has %d HDD mount(s), hasHDD=%v", stackName, len(hddMounts), hasHDD)
}
// Build list of paths to restore from the snapshot
var restorePaths []string
// Always restore the stack's config dir (compose + app.yaml + .felhom.yml)
composePath, ok := m.stackProvider.GetStackComposePath(stackName)
if ok {
stackDir := filepath.Dir(composePath)
restorePaths = append(restorePaths, stackDir)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore config dir: %s", stackDir)
}
}
// Restore DB dump files for this stack (per-drive path)
drivePath := m.GetAppDrivePath(stackName)
dumpDir := AppDBDumpPath(drivePath, stackName)
restorePaths = append(restorePaths, dumpDir)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore DB dump dir: %s", dumpDir)
if drivePath == "" {
return fmt.Errorf("cannot determine drive path for %s", stackName)
}
// Restore HDD data (always included for apps that have it — backup is mandatory)
if hasHDD {
restorePaths = append(restorePaths, hddMounts...)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: will restore HDD data: %v", hddMounts)
}
}
// Restore Docker volume dumps (if present in snapshot)
volDumpDir := AppVolumeDumpPath(drivePath, stackName)
restorePaths = append(restorePaths, volDumpDir)
if len(restorePaths) == 0 {
return fmt.Errorf("no restorable paths found for %s", stackName)
}
// Use the app's primary restic repo
repoPath := PrimaryResticRepoPath(drivePath)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: using repo=%s, %d restore path(s)", repoPath, len(restorePaths))
}
m.logger.Printf("[INFO] [backup] Starting restore for %s (snapshot=%s, repo=%s, paths=%v, hasHDD=%v)",
stackName, snapshotID, repoPath, restorePaths, hasHDD)
m.logger.Printf("[INFO] [backup] Starting app-data restore for %s (drive=%s)", stackName, drivePath)
// Stop the app before restore
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 1/4 — stopping app %s", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 1/3 — stopping app %s", stackName)
}
if err := m.stackProvider.StopStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not stop %s: %v (proceeding anyway)", stackName, err)
}
// Execute restore via restic
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 2/4 — restoring data from snapshot %s", snapshotID)
}
if err := m.restic.RestoreAppData(repoPath, snapshotID, restorePaths); err != nil {
m.logger.Printf("[ERROR] RESTORE failed for %s: %v", stackName, err)
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 3/4 — restarting app %s after failure", stackName)
}
if startErr := m.stackProvider.StartStack(stackName); startErr != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after failure: %v", stackName, startErr)
}
return err
}
// Populate Docker volumes from restored tars
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 3/5 — restoring Docker volumes for %s", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 2/3 — restoring Docker volumes for %s", stackName)
}
if err := m.restoreDockerVolumes(stackName, drivePath); err != nil {
m.logger.Printf("[WARN] RESTORE volume restore failed for %s: %v (continuing)", stackName, err)
@@ -132,7 +66,7 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
// Restart the app
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 4/5 — restarting app %s after successful restore", stackName)
m.logger.Printf("[DEBUG] RestoreApp: step 3/3 — restarting app %s after restore", stackName)
}
if err := m.stackProvider.StartStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after restore: %v", stackName, err)
@@ -143,219 +77,7 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
m.logger.Printf("[WARN] [backup] Restore completed but app health check failed: %v", err)
}
hasVolumes := len(m.stackProvider.GetDockerVolumes(stackName)) > 0
restoreType := "config+DB"
if hasHDD || hasVolumes {
restoreType = "full (config+DB+userdata)"
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 5/5 — restore completed, type=%s", restoreType)
}
m.logger.Printf("[INFO] RESTORE completed: stack=%s, snapshot=%s, type=%s", stackName, snapshotID, restoreType)
return nil
}
// RestoreAppFromTier2 restores an app from its cross-drive rsync backup mirror.
func (m *Manager) RestoreAppFromTier2(stackName string) error {
if m.stackProvider == nil {
return fmt.Errorf("stack provider not configured")
}
if m.settings == nil {
return fmt.Errorf("settings not available")
}
cdCfg := m.settings.GetCrossDriveConfig(stackName)
if cdCfg == nil || !cdCfg.Enabled {
return fmt.Errorf("cross-drive backup not configured for %s", stackName)
}
rsyncDir := AppSecondaryRsyncPath(cdCfg.DestinationPath, stackName)
if _, err := os.Stat(rsyncDir); os.IsNotExist(err) {
return fmt.Errorf("Tier 2 backup directory not found: %s", rsyncDir)
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: stack=%s, rsyncDir=%s", stackName, rsyncDir)
}
// Prevent concurrent operations
m.mu.Lock()
if m.running {
m.mu.Unlock()
return fmt.Errorf("backup or restore already in progress")
}
m.running = true
m.mu.Unlock()
defer func() {
m.mu.Lock()
m.running = false
m.mu.Unlock()
}()
hddMounts := m.stackProvider.GetStackHDDMounts(stackName)
hasHDD := len(hddMounts) > 0
drivePath := m.GetAppDrivePath(stackName)
m.logger.Printf("[INFO] [backup] Starting Tier 2 restore for %s from %s", stackName, rsyncDir)
// Step 1: Stop the app
if err := m.stackProvider.StopStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not stop %s: %v (proceeding anyway)", stackName, err)
}
// Step 2: Restore config from _config/
configSrc := filepath.Join(rsyncDir, "_config") + "/"
if _, err := os.Stat(filepath.Join(rsyncDir, "_config")); err == nil {
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok {
configDst := filepath.Dir(composePath) + "/"
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync config %s → %s", configSrc, configDst)
}
cmd := exec.Command("rsync", "-a", "--delete", configSrc, configDst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 config restore failed for %s: %v (%s)", stackName, err, strings.TrimSpace(string(out)))
// Try to restart and return error
m.stackProvider.StartStack(stackName)
return fmt.Errorf("config restore failed: %w", err)
}
}
}
// Step 3: Restore HDD data
if hasHDD {
// Check for data directory structure — single mount vs multi-mount
if len(hddMounts) == 1 {
// Single mount: data is directly in rsyncDir (excluding _* dirs)
src := strings.TrimRight(rsyncDir, "/") + "/"
dst := strings.TrimRight(hddMounts[0], "/") + "/"
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync HDD data %s → %s", src, dst)
}
cmd := exec.Command("rsync", "-a", "--delete",
"--exclude", "_*",
src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 HDD data restore failed for %s: %v (%s)", stackName, err, strings.TrimSpace(string(out)))
m.stackProvider.StartStack(stackName)
return fmt.Errorf("HDD data restore failed: %w", err)
}
} else {
// Multiple mounts: each has a subdirectory named by leaf
for _, mount := range hddMounts {
leaf := filepath.Base(mount)
src := filepath.Join(rsyncDir, leaf) + "/"
dst := strings.TrimRight(mount, "/") + "/"
if _, err := os.Stat(filepath.Join(rsyncDir, leaf)); os.IsNotExist(err) {
m.logger.Printf("[WARN] [backup] Tier 2 restore: no backup data for mount %s", mount)
continue
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: rsync HDD mount %s → %s", src, dst)
}
cmd := exec.Command("rsync", "-a", "--delete", src, dst)
if out, err := cmd.CombinedOutput(); err != nil {
m.logger.Printf("[ERROR] [backup] Tier 2 HDD restore failed for mount %s: %v (%s)", mount, err, strings.TrimSpace(string(out)))
m.stackProvider.StartStack(stackName)
return fmt.Errorf("HDD restore failed for %s: %w", mount, err)
}
}
}
}
// Step 4: Restore DB dumps from _db/
dbSrc := filepath.Join(rsyncDir, "_db")
if _, err := os.Stat(dbSrc); err == nil {
dbDst := AppDBDumpPath(drivePath, stackName)
if err := os.MkdirAll(dbDst, 0755); err == nil {
entries, _ := os.ReadDir(dbSrc)
for _, e := range entries {
if !e.IsDir() {
src := filepath.Join(dbSrc, e.Name())
dst := filepath.Join(dbDst, e.Name())
if err := copyFile(src, dst); err != nil {
m.logger.Printf("[WARN] [backup] Failed to copy DB dump %s: %v", e.Name(), err)
}
}
}
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreAppFromTier2: restored DB dumps from %s", dbSrc)
}
}
}
// Step 5: Restore Docker volumes from _volumes/
volSrc := filepath.Join(rsyncDir, "_volumes")
if _, err := os.Stat(volSrc); err == nil {
if err := m.restoreDockerVolumesFromDir(stackName, volSrc); err != nil {
m.logger.Printf("[WARN] [backup] Tier 2 volume restore failed for %s: %v (continuing)", stackName, err)
}
}
// Step 6: Restart the app
if err := m.stackProvider.StartStack(stackName); err != nil {
m.logger.Printf("[WARN] RESTORE could not restart %s after Tier 2 restore: %v", stackName, err)
}
// Verify app started successfully
if err := m.waitForHealthy(stackName, 90*time.Second); err != nil {
m.logger.Printf("[WARN] [backup] Tier 2 restore completed but app health check failed: %v", err)
}
hasVolumes := len(m.stackProvider.GetDockerVolumes(stackName)) > 0
restoreType := "config+DB"
if hasHDD || hasVolumes {
restoreType = "full (config+DB+userdata)"
}
m.logger.Printf("[INFO] RESTORE (Tier 2) completed: stack=%s, type=%s", stackName, restoreType)
return nil
}
// restoreDockerVolumesFromDir populates Docker volumes from tar files in an arbitrary directory.
// Used by Tier 2 restore where volume tars are in the rsync mirror's _volumes/ dir.
func (m *Manager) restoreDockerVolumesFromDir(stackName, dumpDir string) error {
entries, err := os.ReadDir(dumpDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("reading volume dump dir: %w", err)
}
var restored int
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tar") {
continue
}
volName := strings.TrimSuffix(entry.Name(), ".tar")
m.logger.Printf("[INFO] [backup] Restoring Docker volume %s for %s (Tier 2)", volName, stackName)
exec.Command("docker", "volume", "rm", "-f", volName).Run()
if out, err := exec.Command("docker", "volume", "create", volName).CombinedOutput(); err != nil {
m.logger.Printf("[WARN] [backup] Failed to create volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err)
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
"-v", volName+":/vol",
"-v", dumpDir+":/in:ro",
"alpine", "tar", "xf", "/in/"+entry.Name(), "-C", "/vol")
out, err := cmd.CombinedOutput()
cancel()
if err != nil {
m.logger.Printf("[WARN] [backup] Failed to populate volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err)
continue
}
restored++
}
if restored > 0 {
m.logger.Printf("[INFO] [backup] Restored %d Docker volume(s) for %s (Tier 2)", restored, stackName)
}
m.logger.Printf("[INFO] RESTORE completed: stack=%s", stackName)
return nil
}
@@ -416,7 +138,6 @@ func (m *Manager) restoreDockerVolumes(stackName, drivePath string) error {
// waitForHealthy waits for a stack to reach running state after restore.
// Forces a docker ps refresh on each poll to avoid stale state.
// Acceptable overhead for a rare operation (restore).
func (m *Manager) waitForHealthy(stackName string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
interval := 5 * time.Second