package backup import ( "context" "fmt" "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" "gitea.dooplex.hu/admin/felhom-controller/internal/util" "os" "strings" "time" ) // RestoreApp restores an app's data from its on-disk app-data backup. // // Disk-tier (restic snapshot) restore has moved to the host agent. This keep-side // restore re-imports the Docker-volume tar dumps that the app-data backup produced // (AppVolumeDumpPath) and relies on the DB dumps already present on the app's drive. // The stack is stopped before the volume import and restarted after. // // R-638: the order is stop → volumes → DB-only start → replay → full start, the same as the unit // restore (R-47). The replay is skipped when the volume leg failed, and a dump with no identifiable // database service is refused before anything is touched. // // snapshotID is retained for API/UI signature compatibility; with restic removed it // is only used for logging (the source of truth is now the on-disk volume tars). func (m *Manager) RestoreApp(stackName, snapshotID string) error { if m.stackProvider == nil { return fmt.Errorf("stack provider not configured") } if m.isDebug() { m.logger.Printf("[DEBUG] RestoreApp: stack=%s, snapshotID=%s", stackName, snapshotID) } // Prevent concurrent operations m.mu.Lock() if m.running { m.mu.Unlock() return fmt.Errorf("backup or restore already in progress") } m.running = true m.mu.Unlock() defer func() { m.mu.Lock() m.running = false m.mu.Unlock() }() drivePath := m.GetAppDrivePath(stackName) if drivePath == "" { return fmt.Errorf("cannot determine drive path for %s", stackName) } m.logger.Printf("[INFO] [backup] Starting app-data restore for %s (drive=%s)", stackName, drivePath) // R-638 (option A, 09 §3 decision 154): which compose service holds the database, and is there // anything to replay? Resolved BEFORE the first mutation, from the LIVE compose — this fallback has no // unit, so the live definition is the one that will run — exactly as the off-site path does for an app // that keeps its definition (offbox_reconstitute.go, "WHICH SERVICE HOLDS THE DATABASE"). dumpDir := AppDBDumpPath(m.namespaceRoot(drivePath), stackName) hasDumps := hasReplayableDump(dumpDir) var dbServices []string if hasDumps { if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok && composePath != "" { svcs, dsErr := DBServiceNames(composePath) if dsErr != nil { // "cannot tell" is not "no database" — leave it empty and let the gate below refuse. m.logger.Printf("[WARN] [backup] %s: could not read the live compose services: %v", stackName, dsErr) } dbServices = svcs } // Fail-closed, the same gate as the unit and off-site paths: the only alternative to refusing is // to start the WHOLE stack and replay into it — the R-638 shape, where a newer app migrates the // restored data before the old copy is poured over it. Refused with the live app untouched. // Pinned by TestR638_FallbackRefusesWhenNoDBServiceIdentifiable. if len(dbServices) == 0 { m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s: a .sql dump exists but no database service is identifiable in the live compose", stackName) return util.MsgError("err.backup.az_adatbazis_szolgaltatas_nem_azonosithato_a", stackName) } } // Stop the app before restore if m.isDebug() { m.logger.Printf("[DEBUG] RestoreApp: step 1/4 — stopping app %s", stackName) } if err := m.stackProvider.StopStack(stackName); err != nil { m.logger.Printf("[WARN] RESTORE could not stop %s: %v (proceeding anyway)", stackName, err) } // F17: surface a data-restore failure instead of swallowing it. We still bring the app back up so it // isn't left dead, but the error is returned at the end so a failed restore can't read as success. var dataErr error // Populate Docker volumes from restored tars if m.isDebug() { m.logger.Printf("[DEBUG] RestoreApp: step 2/4 — restoring Docker volumes for %s", stackName) } _, volErr := m.restoreDockerVolumes(stackName, drivePath) if volErr != nil { m.logger.Printf("[ERROR] RESTORE volume restore failed for %s: %v", stackName, volErr) dataErr = volErr } // F17: replay the captured .sql dump (the legacy path never did this, so DB-resident data did not // come back). Runs after the volume restore so the dump WINS over any tar copy. // // R-638: with ONLY the database service up — the same DB-only start the unit restore uses (R-47). // This used to run after a FULL StartStack at the CURRENT definition, which gave a newer app the // window to migrate the just-restored data; the loader then overlays the copy and removes only what // the copy knows about, so the newer tables stayed behind (measured 2026-09-23, romm 5.0 → 5.3). // Pinned by TestR638_FallbackReplaysBeforeTheAppStarts. // // And NOT when the volume leg failed: the database's volume is then not known to hold the copy's // own files (it may still be the live, newer one), and the overlay would leave the difference // behind. The failure is already in dataErr and is returned below. Pinned by // TestR638_FallbackVolumeFailureSkipsTheReplay. if hasDumps { if volErr != nil { m.logger.Printf("[ERROR] [backup] RESTORE %s: NOT replaying the database copy — the volume restore failed, so the database is not known to be the copy's own", stackName) } else { if m.isDebug() { m.logger.Printf("[DEBUG] RestoreApp: step 3/4 — starting only %v and replaying the DB copy for %s", dbServices, stackName) } if err := m.stackProvider.StartStackServices(stackName, dbServices); err != nil { m.logger.Printf("[ERROR] [backup] DB-only start for %s: %v", stackName, err) if dataErr == nil { dataErr = err } } else if _, err := m.reimportDBDumpsCtx(stackName, m.namespaceRoot(drivePath)); err != nil { m.logger.Printf("[ERROR] RESTORE DB re-import failed for %s: %v", stackName, err) if dataErr == nil { dataErr = err } } } } // Restart the app — every exit from the DB-only window ends in a full start. if m.isDebug() { m.logger.Printf("[DEBUG] RestoreApp: step 4/4 — starting app %s after restore", stackName) } if err := m.stackProvider.StartStack(stackName); err != nil { m.logger.Printf("[WARN] RESTORE could not restart %s after restore: %v", stackName, err) } // Verify app started successfully if err := m.waitForHealthy(stackName, healthTimeout); err != nil { m.logger.Printf("[WARN] [backup] Restore completed but app health check failed: %v", err) } if dataErr != nil { return fmt.Errorf("restore of %s completed with data errors: %w", stackName, dataErr) } m.logger.Printf("[INFO] RESTORE completed: stack=%s", stackName) return nil } // restoreDockerVolumes populates Docker volumes from the tars in the app's LIVE recovery unit, and // returns HOW MANY it replayed. // // R-353: the count used to be discarded here. `restoreDockerVolumesFrom` has always returned it, so // the fact existed one call deep and was thrown away one line later — which left the unit-restore // path structurally unable to tell a customer whether any data came back. On 2026-08-21 an opengist // restore reported completion over a unit holding manifest.json and compose/ and nothing else, and no // screen could have said otherwise. Discarding a fact the caller needs is cheaper to fix than to // re-derive: the caller cannot count volumes afterwards without re-reading the directory the restore // has already consumed. func (m *Manager) restoreDockerVolumes(stackName, drivePath string) (int, error) { // R-638: through the same volume-replay seam the unit and off-site paths use (R-354), so the // fallback's ordering can be tested without a Docker daemon. Nil in production. replay := m.volumeReplayFrom if replay == nil { replay = m.restoreDockerVolumesFrom } return replay(stackName, AppVolumeDumpPath(m.namespaceRoot(drivePath), stackName)) } // restoreDockerVolumesFrom is restoreDockerVolumes with an EXPLICIT dump directory, and it returns how // many volumes it replayed. // // R-354. The off-site reconstitution needs exactly this, for the same reason reimportDBDumpsFrom // exists beside reimportDBDumps: the snapshot's archives live under the restored SCRATCH unit, because // the live unit is deliberately never overwritten by a placement. Until now no such variant existed, // so the off-site path had no way to replay a volume and simply did not — the tar sat in the unit, in // the snapshot and in the verification folder, and the restore reported success without it. For an app // whose data is entirely in a named volume — 40 of the 53 in the catalogue — that is everything the // customer owns. // // ONE implementation, two callers. A second copy of this loop is what produced the divergence in the // first place: the local path replayed volumes and the off-site path did not, and nothing compared the // two. // // It only ever READS dumpDir; the recovery unit is never written to here, on either path. func (m *Manager) restoreDockerVolumesFrom(stackName, dumpDir string) (int, error) { entries, err := os.ReadDir(dumpDir) if err != nil { if os.IsNotExist(err) { return 0, nil // No volume dumps to restore } return 0, fmt.Errorf("reading volume dump dir: %w", err) } var restored int var failed []string composeVer := "" for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tar") { continue } volName := strings.TrimSuffix(entry.Name(), ".tar") if composeVer == "" { composeVer = composeVersionShort() } m.logger.Printf("[INFO] [backup] Restoring Docker volume %s for %s", volName, stackName) // Remove existing volume (ignore errors — may not exist) dockerexec.Command("docker", "volume", "rm", "-f", volName).Run() // Create fresh volume — WITH the labels compose gives its own volumes (R-658, v0.268.0). A bare // `volume create` made a volume the update's undo (until v0.267.0) could not find, and one // compose warns it "was not created by Docker Compose". The config-hash label is deliberately // NOT set: compose only compares it when present, and a guessed hash would make it offer to // recreate (empty) the volume. args := append([]string{"volume", "create"}, composeVolumeLabelArgs(stackName, volName, composeVer)...) if len(args) == 2 { m.logger.Printf("[WARN] [backup] volume %s does not carry the %s_ prefix — created WITHOUT compose labels (the undo finds it by the app's definition anyway)", volName, stackName) } args = append(args, volName) if out, err := dockerexec.Command("docker", args...).CombinedOutput(); err != nil { m.logger.Printf("[ERROR] [backup] Failed to create volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err) failed = append(failed, volName) continue } // Populate from tar ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) cmd := dockerexec.CommandContext(ctx, "docker", "run", "--rm", "-v", volName+":/vol", "-v", dumpDir+":/in:ro", "alpine", "tar", "xf", "/in/"+entry.Name(), "-C", "/vol") out, err := cmd.CombinedOutput() cancel() if err != nil { m.logger.Printf("[ERROR] [backup] Failed to populate volume %s: %s — %v", volName, strings.TrimSpace(string(out)), err) failed = append(failed, volName) continue } restored++ if m.isDebug() { m.logger.Printf("[DEBUG] [backup] Volume %s restored successfully", volName) } } if restored > 0 { m.logger.Printf("[INFO] [backup] Restored %d Docker volume(s) for %s", restored, stackName) } // F17: a per-volume failure used to be a swallowed WARN; surface it so the restore is reported as // failed rather than silently partial. The count is returned ALONGSIDE the error, not instead of // it: a caller that replayed three of four volumes needs both numbers to say what happened. if len(failed) > 0 { return restored, fmt.Errorf("failed to restore %d volume(s): %v", len(failed), failed) } return restored, nil } // waitForHealthy's clock (R-488). These are the PRODUCTION values; they are variables only so the // package's tests can shorten them in ONE place (main_test.go TestMain) instead of every restore // test sleeping 3 s of settle and every not-running fixture waiting out the full 90 s — that was // ~280 s of the package's ~440 s. TestR488_HealthWaitDefaultsAreProduction pins the values. var ( healthSettle = 3 * time.Second // initial settling time before the first poll healthInterval = 5 * time.Second // between polls healthTimeout = 90 * time.Second // how long a restored stack gets to reach running ) // waitForHealthy waits for a stack to reach running state after restore. // Forces a docker ps refresh on each poll to avoid stale state. func (m *Manager) waitForHealthy(stackName string, timeout time.Duration) error { deadline := time.Now().Add(timeout) interval := healthInterval time.Sleep(healthSettle) // initial settling time for time.Now().Before(deadline) { if m.stackProvider == nil { return fmt.Errorf("no stack provider") } if m.stackProvider.RefreshAndIsRunning(stackName) { if m.isDebug() { m.logger.Printf("[DEBUG] [backup] Post-restore health check: %s is running", stackName) } return nil } if m.isDebug() { m.logger.Printf("[DEBUG] [backup] Post-restore health check: %s not yet running, waiting...", stackName) } time.Sleep(interval) } return fmt.Errorf("stack %s did not reach running state within %s after restore", stackName, timeout) } // composeVolumeLabelArgs returns the `--label` arguments that make a restored volume look like one // compose created for this project: project, volume key and compose version. The key is the part of // the Docker name after `_`, which is how compose names a volume without its own `name:` — // every catalog template today (R-658, measured 2026-09-24). A name without that prefix gets no labels // rather than a guessed key. The version label is left out when the version could not be read. func composeVolumeLabelArgs(project, volName, composeVersion string) []string { key := strings.TrimPrefix(volName, project+"_") if key == volName || key == "" { return nil } args := []string{"--label", "com.docker.compose.project=" + project, "--label", "com.docker.compose.volume=" + key} if composeVersion != "" { args = append(args, "--label", "com.docker.compose.version="+composeVersion) } return args } // composeVersionShort is `docker compose version --short`, "" when it cannot be read. func composeVersionShort() string { out, err := dockerexec.Command("docker", "compose", "version", "--short").Output() if err != nil { return "" } return strings.TrimPrefix(strings.TrimSpace(string(out)), "v") }