v0.153.0 — R-47: the DB replay no longer races the app, on BOTH restore paths
Closes R-47. No new agent coupling — MinAgent stays 0.90.0. The replay needs a running DB container, so both restore paths started the WHOLE stack first, giving the application a window to rebuild the very schema objects the dump was about to create. Measured live on 2026-07-19 (H4, DIAG-immich-restore-round2): immich-server rebuilt clip_index two seconds before the dump's CREATE INDEX, the replay aborted "already exists" under ON_ERROR_STOP=1, and immich reported schema drift. The data survived only because pg_dump emits COPY before CREATE INDEX. Both paths now open a DB-ONLY window: only the stack's database service(s) come up, the dump is replayed with the app still down, and the full start runs only after the replay exits 0. Fail-closed: a dump with no identifiable DB service refuses BEFORE the first mutation. Every exit from the window still does a best-effort full start, so a failed restore never leaves a box with a database and no application. New: appbackup.DBServiceNames (yaml.v3 services-map parse — never a line scan; immich's top-level volume keys are the decoy) sharing dbTypeForImage with DiscoverDatabases; stacks.Manager.StartStackServices (refuses an empty list — argument-less `up -d` is a full start); RedeployFromEnv split into PersistUnitRedeployConfig + its unchanged tail. StackDataProvider's RecreateStackFromUnit becomes RecreateStackDefinitionFromUnit — the hidden `up -d` inside the old name is what carried the defect on the local path. 19 new tests (ordering plus state-at-replay-time, zero-mutation fail-closed effects, replay-failure bring-up, parser decoys, empty-list refusal); three companion red-proofs run and reverted. 23/23 packages green. Not yet live-validated: STOP-1 supervised reconstitute, golden 0.153.0.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -64,6 +65,26 @@ func readStrippedEnv(path string) map[string]string {
|
||||
return s.Env
|
||||
}
|
||||
|
||||
// hasReplayableDump reports whether dumpDir holds a .sql dump that the replay could actually use.
|
||||
// The `pre-restore-` safety dumps are EXCLUDED: they live in the same directory (deliberately — an
|
||||
// undo the customer cannot see is not much of one) but are never a replay source, so counting them
|
||||
// would arm the DB-only phase, and its fail-closed gate, for an app that has nothing to replay.
|
||||
func hasReplayableDump(dumpDir string) bool {
|
||||
entries, err := os.ReadDir(dumpDir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || filepath.Ext(e.Name()) != ".sql" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(e.Name(), preRestoreDumpPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RestoreFromRecoveryUnit recreates an app from its on-drive recovery unit + the guest's own secrets.
|
||||
//
|
||||
// It reads the unit manifest, recovers the secret values from the guest's live app.yaml, applies the
|
||||
@@ -148,7 +169,22 @@ func (m *Manager) RestoreFromRecoveryUnit(stackName string) error {
|
||||
m.logger.Printf("[INFO] [backup] Restoring %s from recovery unit: images=%d, secrets recovered=%d/%d, data_keys=%d",
|
||||
stackName, len(manifest.ImagePins), len(manifest.SecretEnvVars)-len(missing), len(manifest.SecretEnvVars), len(manifest.DataKeyEnvVars))
|
||||
|
||||
// Stop, restore named-volume data, then recreate the definition + redeploy with the recovered env.
|
||||
// R-47: which compose service holds the database, and is there anything to replay? Resolved from
|
||||
// the UNIT's compose, because that file is about to BECOME the live one. Both answers are needed
|
||||
// BEFORE the first mutation, so the refusal below leaves the live app completely untouched.
|
||||
dbServices, dsErr := DBServiceNames(filepath.Join(composeDir, "docker-compose.yml"))
|
||||
if dsErr != nil {
|
||||
// "cannot tell" is not "no database" — leave it empty and let the gate decide.
|
||||
m.logger.Printf("[WARN] [backup] %s: could not read the unit's compose services: %v", stackName, dsErr)
|
||||
}
|
||||
hasDumps := hasReplayableDump(AppDBDumpPath(nsRoot, stackName))
|
||||
if hasDumps && len(dbServices) == 0 {
|
||||
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s: a .sql dump exists but no database service is identifiable in the unit's compose", stackName)
|
||||
return fmt.Errorf("Az adatbázis-szolgáltatás nem azonosítható a(z) %s alkalmazásban — a visszaállítás biztonsági okból nem indult el.", stackName)
|
||||
}
|
||||
|
||||
// Stop, restore named-volume data, recreate the definition, replay the DB with ONLY the database
|
||||
// service running, and only then start the whole stack.
|
||||
// F17: surface a data-restore failure instead of swallowing it (we still bring the app back up).
|
||||
var dataErr error
|
||||
if err := m.stackProvider.StopStack(stackName); err != nil {
|
||||
@@ -158,17 +194,30 @@ func (m *Manager) RestoreFromRecoveryUnit(stackName string) error {
|
||||
m.logger.Printf("[ERROR] [backup] volume restore for %s: %v", stackName, err)
|
||||
dataErr = err
|
||||
}
|
||||
if err := m.stackProvider.RecreateStackFromUnit(stackName, composeDir, fullEnv); err != nil {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stackName, composeDir, fullEnv); err != nil {
|
||||
return fmt.Errorf("recreating %s from unit: %w", stackName, err)
|
||||
}
|
||||
// F17: the captured .sql dump is the authoritative logical DB state — replay it into the now-running
|
||||
// DB container AFTER the volume restore, so the dump WINS over any volume-tar copy of the database.
|
||||
if _, err := m.reimportDBDumpsCtx(stackName, nsRoot); err != nil {
|
||||
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v", stackName, err)
|
||||
if dataErr == nil {
|
||||
dataErr = err
|
||||
// F17: the captured .sql dump is the authoritative logical DB state — replay it AFTER the volume
|
||||
// restore, so the dump WINS over any volume-tar copy of the database.
|
||||
// R-47: the replay happens with ONLY the database service up. This used to run after
|
||||
// RecreateStackFromUnit had already brought the WHOLE stack up, letting the application rebuild
|
||||
// schema objects underneath the replay (H4, DIAG-immich-restore-round2-2026-07-19).
|
||||
if hasDumps {
|
||||
if err := m.stackProvider.StartStackServices(stackName, dbServices); err != nil {
|
||||
m.logger.Printf("[ERROR] [backup] DB-only start for %s: %v", stackName, err)
|
||||
if dataErr == nil {
|
||||
dataErr = err
|
||||
}
|
||||
} else if _, err := m.reimportDBDumpsCtx(stackName, nsRoot); err != nil {
|
||||
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v", stackName, err)
|
||||
if dataErr == nil {
|
||||
dataErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := m.stackProvider.StartStack(stackName); err != nil {
|
||||
return fmt.Errorf("starting %s after restore from unit: %w", stackName, err)
|
||||
}
|
||||
if err := m.waitForHealthy(stackName, 90*time.Second); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] %s restored but health check failed: %v", stackName, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user