v0.15.5: Disaster recovery — Hub-based infra backup, auto-mount, restore UI
Complete DR implementation (TASK2.md Phases 1-4): - Hub infra-backup push/pull endpoints (controller.yaml, disk layout, stacks) - Fresh-deployment detection pulls config from Hub, auto-mounts drives by UUID - Full-page restore UI with drive status, app table, sequential restore - docker-setup.sh shows DR instructions when customer_id is configured New files: disk_layout.go, restore_scan.go, restore_app_linux.go, restore_drives_linux.go, infra_backup.go, infra_pull.go, handler_restore.go, restore.html Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// InfraBackup is the payload pushed to the Hub for disaster recovery.
|
||||
type InfraBackup struct {
|
||||
CustomerID string `json:"customer_id"`
|
||||
Domain string `json:"domain"`
|
||||
ControllerVersion string `json:"controller_version"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
|
||||
ControllerConfigB64 string `json:"controller_config_b64"`
|
||||
SettingsJSONB64 string `json:"settings_json_b64,omitempty"`
|
||||
|
||||
DiskLayout backup.DiskLayout `json:"disk_layout"`
|
||||
DeployedStacks []InfraStack `json:"deployed_stacks"`
|
||||
|
||||
ResticPassword string `json:"restic_password,omitempty"`
|
||||
CrossDrivePassword string `json:"cross_drive_password,omitempty"`
|
||||
}
|
||||
|
||||
// InfraStack identifies a deployed app for disaster recovery.
|
||||
type InfraStack struct {
|
||||
Name string `json:"name"`
|
||||
DisplayName string `json:"display_name"`
|
||||
HDDPath string `json:"hdd_path,omitempty"`
|
||||
NeedsHDD bool `json:"needs_hdd"`
|
||||
}
|
||||
|
||||
// BuildInfraBackup collects all infrastructure state for Hub backup.
|
||||
func BuildInfraBackup(
|
||||
customerID, domain, version string,
|
||||
controllerYAMLPath string,
|
||||
settingsPath string,
|
||||
resticPasswordFile string,
|
||||
systemDataPath string,
|
||||
sett *settings.Settings,
|
||||
stackProvider backup.StackDataProvider,
|
||||
) (*InfraBackup, error) {
|
||||
ib := &InfraBackup{
|
||||
CustomerID: customerID,
|
||||
Domain: domain,
|
||||
ControllerVersion: version,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
// Read and encode controller.yaml
|
||||
if data, err := os.ReadFile(controllerYAMLPath); err == nil {
|
||||
ib.ControllerConfigB64 = base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// Read and encode settings.json
|
||||
if data, err := os.ReadFile(settingsPath); err == nil {
|
||||
ib.SettingsJSONB64 = base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// Read primary restic password
|
||||
if data, err := os.ReadFile(resticPasswordFile); err == nil {
|
||||
ib.ResticPassword = base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// Read cross-drive restic password
|
||||
if pw := sett.GetCrossDriveResticPassword(); pw != "" {
|
||||
ib.CrossDrivePassword = pw
|
||||
}
|
||||
|
||||
// Collect disk layout from fstab + blkid
|
||||
ib.DiskLayout = collectDiskLayout(systemDataPath)
|
||||
|
||||
// Collect deployed stacks
|
||||
deployed := stackProvider.ListDeployedStacks()
|
||||
for _, s := range deployed {
|
||||
ib.DeployedStacks = append(ib.DeployedStacks, InfraStack{
|
||||
Name: s.Name,
|
||||
DisplayName: s.DisplayName,
|
||||
HDDPath: stackProvider.GetStackHDDPath(s.Name),
|
||||
NeedsHDD: s.NeedsHDD,
|
||||
})
|
||||
}
|
||||
if ib.DeployedStacks == nil {
|
||||
ib.DeployedStacks = []InfraStack{}
|
||||
}
|
||||
|
||||
return ib, nil
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//go:build linux
|
||||
|
||||
package report
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
)
|
||||
|
||||
// collectDiskLayout reads /host-fstab and correlates with blkid/lsblk to build
|
||||
// the disk mount topology. Only includes data partitions (not root, boot, or swap).
|
||||
func collectDiskLayout(systemDataPath string) backup.DiskLayout {
|
||||
layout := backup.DiskLayout{}
|
||||
|
||||
fstabPath := "/host-fstab"
|
||||
if _, err := os.Stat(fstabPath); err != nil {
|
||||
fstabPath = "/etc/fstab"
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(fstabPath)
|
||||
if err != nil {
|
||||
return layout
|
||||
}
|
||||
|
||||
// Parse fstab into UUID-based entries and bind mount entries
|
||||
type fstabEntry struct {
|
||||
source string
|
||||
mountPoint string
|
||||
fsType string
|
||||
options string
|
||||
}
|
||||
|
||||
var uuidEntries []fstabEntry
|
||||
var bindEntries []fstabEntry
|
||||
|
||||
systemMounts := map[string]bool{"/": true, "/boot": true, "/boot/efi": true}
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 4 {
|
||||
continue
|
||||
}
|
||||
source := fields[0]
|
||||
mountPoint := fields[1]
|
||||
fsType := fields[2]
|
||||
options := fields[3]
|
||||
|
||||
// Skip system mounts and swap
|
||||
if systemMounts[mountPoint] || fsType == "swap" {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(source, "UUID=") {
|
||||
uuidEntries = append(uuidEntries, fstabEntry{
|
||||
source: strings.TrimPrefix(source, "UUID="),
|
||||
mountPoint: mountPoint,
|
||||
fsType: fsType,
|
||||
options: options,
|
||||
})
|
||||
} else if fsType == "none" && strings.Contains(options, "bind") {
|
||||
bindEntries = append(bindEntries, fstabEntry{
|
||||
source: source,
|
||||
mountPoint: mountPoint,
|
||||
options: options,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Process UUID-based entries
|
||||
for _, e := range uuidEntries {
|
||||
dm := backup.DiskMount{
|
||||
UUID: e.source,
|
||||
MountPoint: e.mountPoint,
|
||||
FSType: e.fsType,
|
||||
FstabOptions: e.options,
|
||||
}
|
||||
|
||||
// Get label via blkid
|
||||
if out, err := exec.Command("blkid", "-o", "value", "-s", "LABEL", "-U", e.source).Output(); err == nil {
|
||||
dm.Label = strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// Get size via lsblk (resolve UUID to device first)
|
||||
if devPath, err := exec.Command("blkid", "-U", e.source).Output(); err == nil {
|
||||
dev := strings.TrimSpace(string(devPath))
|
||||
if dev != "" {
|
||||
if out, err := exec.Command("lsblk", "-b", "-n", "-o", "SIZE", dev).Output(); err == nil {
|
||||
if sz, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64); err == nil {
|
||||
dm.SizeBytes = sz
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine role
|
||||
if e.mountPoint == systemDataPath {
|
||||
dm.Role = "system_data"
|
||||
} else {
|
||||
dm.Role = "hdd_storage"
|
||||
}
|
||||
|
||||
// Check for a corresponding bind mount
|
||||
for _, bind := range bindEntries {
|
||||
if strings.HasPrefix(bind.source, e.mountPoint+"/") {
|
||||
subdir := strings.TrimPrefix(bind.source, e.mountPoint+"/")
|
||||
dm.BindSubdir = subdir
|
||||
dm.RawMount = e.mountPoint
|
||||
dm.MountPoint = bind.mountPoint // the final user-facing mount point
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Get label from mount point basename as fallback
|
||||
if dm.Label == "" {
|
||||
if dm.RawMount != "" {
|
||||
dm.Label = filepath.Base(dm.RawMount)
|
||||
} else {
|
||||
dm.Label = filepath.Base(dm.MountPoint)
|
||||
}
|
||||
}
|
||||
|
||||
layout.Mounts = append(layout.Mounts, dm)
|
||||
}
|
||||
|
||||
return layout
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//go:build !linux
|
||||
|
||||
package report
|
||||
|
||||
import "gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
|
||||
// collectDiskLayout is a no-op on non-Linux platforms.
|
||||
// The controller only runs on Linux; this stub allows cross-compilation.
|
||||
func collectDiskLayout(systemDataPath string) backup.DiskLayout {
|
||||
return backup.DiskLayout{}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PullInfraBackup fetches the infrastructure backup from the Hub.
|
||||
// Returns nil, nil if no backup exists for this customer.
|
||||
func PullInfraBackup(hubURL, apiKey, customerID string) (*InfraBackup, error) {
|
||||
url := strings.TrimRight(hubURL, "/") + "/api/v1/infra-backup/" + customerID
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hub request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil // no backup for this customer
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("hub returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB limit
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading response: %w", err)
|
||||
}
|
||||
|
||||
var ib InfraBackup
|
||||
if err := json.Unmarshal(body, &ib); err != nil {
|
||||
return nil, fmt.Errorf("parsing infra backup: %w", err)
|
||||
}
|
||||
|
||||
return &ib, nil
|
||||
}
|
||||
@@ -82,6 +82,49 @@ func (p *Pusher) Push(report *Report) error {
|
||||
return fmt.Errorf("hub push failed after 3 attempts: %w", lastErr)
|
||||
}
|
||||
|
||||
// PushInfraBackup sends the infrastructure backup payload to the Hub.
|
||||
// Uses the same retry logic as Push.
|
||||
func (p *Pusher) PushInfraBackup(data []byte) error {
|
||||
if !p.enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
url := p.hubURL + "/api/v1/infra-backup"
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if p.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
}
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
p.logger.Printf("[INFO] Infra backup pushed to Hub (%d bytes)", len(data))
|
||||
return nil
|
||||
}
|
||||
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return fmt.Errorf("infra backup push failed after 3 attempts: %w", lastErr)
|
||||
}
|
||||
|
||||
// PushOnce sends a single report regardless of the enabled flag.
|
||||
// Used for one-time notifications (e.g., reporting-disabled on startup).
|
||||
func (p *Pusher) PushOnce(report *Report) error {
|
||||
|
||||
Reference in New Issue
Block a user