v0.153.0 — R-47: the DB replay no longer races the app, on BOTH restore paths

Closes R-47. No new agent coupling — MinAgent stays 0.90.0.

The replay needs a running DB container, so both restore paths started the
WHOLE stack first, giving the application a window to rebuild the very schema
objects the dump was about to create. Measured live on 2026-07-19 (H4,
DIAG-immich-restore-round2): immich-server rebuilt clip_index two seconds
before the dump's CREATE INDEX, the replay aborted "already exists" under
ON_ERROR_STOP=1, and immich reported schema drift. The data survived only
because pg_dump emits COPY before CREATE INDEX.

Both paths now open a DB-ONLY window: only the stack's database service(s)
come up, the dump is replayed with the app still down, and the full start
runs only after the replay exits 0. Fail-closed: a dump with no identifiable
DB service refuses BEFORE the first mutation. Every exit from the window
still does a best-effort full start, so a failed restore never leaves a box
with a database and no application.

New: appbackup.DBServiceNames (yaml.v3 services-map parse — never a line
scan; immich's top-level volume keys are the decoy) sharing dbTypeForImage
with DiscoverDatabases; stacks.Manager.StartStackServices (refuses an empty
list — argument-less `up -d` is a full start); RedeployFromEnv split into
PersistUnitRedeployConfig + its unchanged tail. StackDataProvider's
RecreateStackFromUnit becomes RecreateStackDefinitionFromUnit — the hidden
`up -d` inside the old name is what carried the defect on the local path.

19 new tests (ordering plus state-at-replay-time, zero-mutation fail-closed
effects, replay-failure bring-up, parser decoys, empty-list refusal); three
companion red-proofs run and reverted. 23/23 packages green.

Not yet live-validated: STOP-1 supervised reconstitute, golden 0.153.0.
This commit is contained in:
2026-07-20 17:01:52 +02:00
parent fd40b29119
commit 78ff991f1c
25 changed files with 1323 additions and 201 deletions
+57 -8
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"gopkg.in/yaml.v3"
@@ -64,6 +65,26 @@ func readStrippedEnv(path string) map[string]string {
return s.Env
}
// hasReplayableDump reports whether dumpDir holds a .sql dump that the replay could actually use.
// The `pre-restore-` safety dumps are EXCLUDED: they live in the same directory (deliberately — an
// undo the customer cannot see is not much of one) but are never a replay source, so counting them
// would arm the DB-only phase, and its fail-closed gate, for an app that has nothing to replay.
func hasReplayableDump(dumpDir string) bool {
entries, err := os.ReadDir(dumpDir)
if err != nil {
return false
}
for _, e := range entries {
if e.IsDir() || filepath.Ext(e.Name()) != ".sql" {
continue
}
if !strings.HasPrefix(e.Name(), preRestoreDumpPrefix) {
return true
}
}
return false
}
// RestoreFromRecoveryUnit recreates an app from its on-drive recovery unit + the guest's own secrets.
//
// It reads the unit manifest, recovers the secret values from the guest's live app.yaml, applies the
@@ -148,7 +169,22 @@ func (m *Manager) RestoreFromRecoveryUnit(stackName string) error {
m.logger.Printf("[INFO] [backup] Restoring %s from recovery unit: images=%d, secrets recovered=%d/%d, data_keys=%d",
stackName, len(manifest.ImagePins), len(manifest.SecretEnvVars)-len(missing), len(manifest.SecretEnvVars), len(manifest.DataKeyEnvVars))
// Stop, restore named-volume data, then recreate the definition + redeploy with the recovered env.
// R-47: which compose service holds the database, and is there anything to replay? Resolved from
// the UNIT's compose, because that file is about to BECOME the live one. Both answers are needed
// BEFORE the first mutation, so the refusal below leaves the live app completely untouched.
dbServices, dsErr := DBServiceNames(filepath.Join(composeDir, "docker-compose.yml"))
if dsErr != nil {
// "cannot tell" is not "no database" — leave it empty and let the gate decide.
m.logger.Printf("[WARN] [backup] %s: could not read the unit's compose services: %v", stackName, dsErr)
}
hasDumps := hasReplayableDump(AppDBDumpPath(nsRoot, stackName))
if hasDumps && len(dbServices) == 0 {
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s: a .sql dump exists but no database service is identifiable in the unit's compose", stackName)
return fmt.Errorf("Az adatbázis-szolgáltatás nem azonosítható a(z) %s alkalmazásban — a visszaállítás biztonsági okból nem indult el.", stackName)
}
// Stop, restore named-volume data, recreate the definition, replay the DB with ONLY the database
// service running, and only then start the whole stack.
// F17: surface a data-restore failure instead of swallowing it (we still bring the app back up).
var dataErr error
if err := m.stackProvider.StopStack(stackName); err != nil {
@@ -158,17 +194,30 @@ func (m *Manager) RestoreFromRecoveryUnit(stackName string) error {
m.logger.Printf("[ERROR] [backup] volume restore for %s: %v", stackName, err)
dataErr = err
}
if err := m.stackProvider.RecreateStackFromUnit(stackName, composeDir, fullEnv); err != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stackName, composeDir, fullEnv); err != nil {
return fmt.Errorf("recreating %s from unit: %w", stackName, err)
}
// F17: the captured .sql dump is the authoritative logical DB state — replay it into the now-running
// DB container AFTER the volume restore, so the dump WINS over any volume-tar copy of the database.
if _, err := m.reimportDBDumpsCtx(stackName, nsRoot); err != nil {
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v", stackName, err)
if dataErr == nil {
dataErr = err
// F17: the captured .sql dump is the authoritative logical DB state — replay it AFTER the volume
// restore, so the dump WINS over any volume-tar copy of the database.
// R-47: the replay happens with ONLY the database service up. This used to run after
// RecreateStackFromUnit had already brought the WHOLE stack up, letting the application rebuild
// schema objects underneath the replay (H4, DIAG-immich-restore-round2-2026-07-19).
if hasDumps {
if err := m.stackProvider.StartStackServices(stackName, dbServices); err != nil {
m.logger.Printf("[ERROR] [backup] DB-only start for %s: %v", stackName, err)
if dataErr == nil {
dataErr = err
}
} else if _, err := m.reimportDBDumpsCtx(stackName, nsRoot); err != nil {
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v", stackName, err)
if dataErr == nil {
dataErr = err
}
}
}
if err := m.stackProvider.StartStack(stackName); err != nil {
return fmt.Errorf("starting %s after restore from unit: %w", stackName, err)
}
if err := m.waitForHealthy(stackName, 90*time.Second); err != nil {
m.logger.Printf("[WARN] [backup] %s restored but health check failed: %v", stackName, err)
}