R-893 slice 1: a failed off-site replay after files/volumes/version moved holds the app
Decision 192 (D8, option C). After a failed off-site database replay whose rollback worked, but where the snapshot's definition was written or a named volume was replaced, the app is no longer started on the mixed state: the live definition is written back, the database service stopped, and the app held (HoldReasonRestoreMixed) for support. The sentence (hu+en) says the app needs help and no longer claims the data is back as it was. The operator gets a backup_run_failures mail (leg restore-hold-mixed). The plain case (no version change, no volume replaced) keeps today's behaviour (TestR379_ScenarioA). Red-proofed: r893_mixed_restore_test.go. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -369,6 +369,9 @@ func (m *Manager) RestoreHoldForLang(stack, lang string) (bool, string) {
|
||||
if t, err := time.Parse(time.RFC3339, h.At); err == nil {
|
||||
when = t.Format("2006-01-02 15:04")
|
||||
}
|
||||
if h.Reason == settings.HoldReasonRestoreMixed { // R-893
|
||||
return true, util.Text(lang, "note.reconstitute.held_mixed", stack, when)
|
||||
}
|
||||
return true, util.Text(lang, "note.reconstitute.held", stack, when)
|
||||
}
|
||||
|
||||
@@ -441,6 +444,93 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
|
||||
}
|
||||
}
|
||||
|
||||
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
|
||||
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
|
||||
type liveDefinition struct {
|
||||
dir string
|
||||
env map[string]string
|
||||
}
|
||||
|
||||
func (d *liveDefinition) cleanup() {
|
||||
if d != nil && d.dir != "" {
|
||||
_ = os.RemoveAll(d.dir)
|
||||
}
|
||||
}
|
||||
|
||||
// captureLiveDefinition copies the live docker-compose.yml (required) and .felhom.yml (when present) and
|
||||
// reads the live env (non-secret values plus the live secrets, decrypted from the guest's app.yaml) — the
|
||||
// same two sources the restore path uses. Values are never logged.
|
||||
func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
|
||||
composePath, ok := m.stackProvider.GetStackComposePath(stack)
|
||||
if !ok || composePath == "" {
|
||||
return nil, fmt.Errorf("no live compose path for %s", stack)
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "felhom-livedef-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &liveDefinition{dir: dir, env: map[string]string{}}
|
||||
for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} {
|
||||
data, rErr := os.ReadFile(filepath.Join(filepath.Dir(composePath), fname))
|
||||
if rErr != nil {
|
||||
if fname == "docker-compose.yml" {
|
||||
d.cleanup()
|
||||
return nil, rErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if wErr := os.WriteFile(filepath.Join(dir, fname), data, 0o600); wErr != nil {
|
||||
d.cleanup()
|
||||
return nil, wErr
|
||||
}
|
||||
}
|
||||
if info, ok := m.stackProvider.GetStackRecoveryInfo(stack); ok {
|
||||
for k, v := range info.NonSecretEnv {
|
||||
d.env[k] = v
|
||||
}
|
||||
for k, v := range m.stackProvider.RecoverStackSecrets(stack, info.SecretEnvVars) {
|
||||
d.env[k] = v
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// holdAppAfterMixedRestore (R-893, `09` §3 decision 192, option C) is the hold for a failed replay whose
|
||||
// database rollback WORKED but which had already moved more than the database: the live definition is
|
||||
// written back (when the snapshot's was written), the database service is stopped again, and the app is
|
||||
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
|
||||
// notification tells this case from R-379's double failure.
|
||||
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
|
||||
if live != nil {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the app is held at the snapshot's definition", stack, err)
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] %s: the live definition is written back", stack)
|
||||
}
|
||||
}
|
||||
if err := m.stackProvider.StopStack(stack); err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: stopping the database service before the hold failed: %v", stack, err)
|
||||
}
|
||||
m.logger.Printf("[ERROR] [offbox] %s: database rolled back, but files/volumes/version had already moved — HOLDING the app stopped for support (R-893); replay error was: %v", stack, replayErr)
|
||||
if m.settings == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
|
||||
return
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
}
|
||||
if err := m.settings.SetRestoreHold(h); err != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: persisting the restore hold FAILED: %v — the app is stopped and unguarded", stack, err)
|
||||
}
|
||||
if m.appStop != nil {
|
||||
m.appStop.End()
|
||||
}
|
||||
if m.restoreHoldNotify != nil {
|
||||
m.restoreHoldNotify(stack, replayErr, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// SetRestoreHoldNotify wires the operator notification for a held app (cmd/controller/main.go).
|
||||
func (m *Manager) SetRestoreHoldNotify(fn func(stack string, replayErr, rollbackErr error)) {
|
||||
m.restoreHoldNotify = fn
|
||||
@@ -732,6 +822,21 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
}
|
||||
}
|
||||
|
||||
// --- THE LIVE DEFINITION, KEPT FOR A FAILED REPLAY (R-893, `09` §3 decision 192) -------------
|
||||
// When the snapshot's definition is about to be written, keep the LIVE one first, so a failed
|
||||
// replay below can write it back before the app is held. Taken before the first mutation; a
|
||||
// failure here refuses with nothing touched — without it the failure branch could not be undone.
|
||||
var liveDef *liveDefinition
|
||||
if defineFromSnapshot {
|
||||
ld, ldErr := m.captureLiveDefinition(stack)
|
||||
if ldErr != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] Restore REFUSED for %s: could not keep the live definition: %v — nothing was touched", stack, ldErr)
|
||||
return res, fmt.Errorf("restoring %s: keeping the live definition failed: %w", stack, ldErr)
|
||||
}
|
||||
liveDef = ld
|
||||
defer liveDef.cleanup()
|
||||
}
|
||||
|
||||
// --- WHICH SERVICE HOLDS THE DATABASE (R-47) ------------------------------------------------
|
||||
// Read from the compose that will RUN: the live one, or — when the app comes back at the snapshot's
|
||||
// version — the snapshot unit's, which is written into the stack dir before the first start. Resolved
|
||||
@@ -907,6 +1012,16 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
return res, util.MsgError("err.backup.db_restore_and_rollback_failed", stack, m.undoCopyPhrase(safetySet))
|
||||
}
|
||||
res.RolledBack = true
|
||||
// --- R-893: A ROLLED-BACK DATABASE IS NOT „BACK AS IT WAS" WHEN MORE MOVED ---------
|
||||
// The rollback puts back the database rows only. When the snapshot's definition was
|
||||
// written or a named volume was replaced, the app would start on a mix: the snapshot's
|
||||
// files and volumes (and maybe its older version) under the newer database. Write the
|
||||
// live definition back, stop the database service again, and HOLD the app for support.
|
||||
// Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA.
|
||||
if defineFromSnapshot || res.VolumesReplayed > 0 {
|
||||
m.holdAppAfterMixedRestore(stack, iErr, liveDef)
|
||||
return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack)
|
||||
}
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
// R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md).
|
||||
//
|
||||
// An off-site restore of one app writes the snapshot's definition (when the version differs), copies the
|
||||
// snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails,
|
||||
// the rollback puts back only the database rows. Until this slice the app was started on that mix —
|
||||
// an older definition, the snapshot's files and volumes, the newer database — and the household was told
|
||||
// „your data is back as it was before the restore". That is true only of the database rows.
|
||||
//
|
||||
// Now: the database is still rolled back, the LIVE definition is written back, and the app is HELD
|
||||
// stopped with a sentence that says it needs support. Pinned here:
|
||||
// (a) the definition write is called a second time, with the live definition;
|
||||
// (b) the app is NOT started;
|
||||
// (c) a hold is left (kind restore_mixed), and the operator is notified;
|
||||
// (d) neither the error nor the hold sentence claims the data is back as it was (hu and en).
|
||||
// The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA.
|
||||
|
||||
// r893Provider records EVERY definition write (vtReconProvider keeps only the last).
|
||||
type r893Provider struct {
|
||||
*vtReconProvider
|
||||
defs [][]string
|
||||
}
|
||||
|
||||
func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error {
|
||||
p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml")))
|
||||
return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env)
|
||||
}
|
||||
|
||||
func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) {
|
||||
t.Helper()
|
||||
m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1))
|
||||
m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") }
|
||||
rolled := 0
|
||||
m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })
|
||||
notified := 0
|
||||
m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) {
|
||||
notified++
|
||||
if replayErr == nil || rollbackErr != nil {
|
||||
t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr)
|
||||
}
|
||||
})
|
||||
m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil }
|
||||
live := "16"
|
||||
if versionChange {
|
||||
live = "18"
|
||||
}
|
||||
// The LIVE definition, on disk where the app runs.
|
||||
if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}}
|
||||
m.SetStackProvider(vp)
|
||||
vtSnapshotUnit(t, m, "16", true)
|
||||
return m, vp, &rolled, ¬ified
|
||||
}
|
||||
|
||||
func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatal("a failed replay must be surfaced as a failure")
|
||||
}
|
||||
if rolled != 1 {
|
||||
t.Fatalf("the database rollback must still run exactly once, ran %d", rolled)
|
||||
}
|
||||
// (b)
|
||||
if vp.fullStarted {
|
||||
t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls)
|
||||
}
|
||||
if last := vp.calls[len(vp.calls)-1]; last != "stop" {
|
||||
t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls)
|
||||
}
|
||||
// (c)
|
||||
held, sentence := m.RestoreHoldFor("immich")
|
||||
if !held {
|
||||
t.Fatal("no hold was left — every start path would start the app on the mixed state")
|
||||
}
|
||||
if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed {
|
||||
t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed)
|
||||
}
|
||||
if notified != 1 {
|
||||
t.Errorf("the operator must be notified once, got %d", notified)
|
||||
}
|
||||
// (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too).
|
||||
for _, s := range []string{err.Error(), sentence} {
|
||||
low := strings.ToLower(s)
|
||||
if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") {
|
||||
t.Errorf("the sentence still claims the data is back / the app runs: %q", s)
|
||||
}
|
||||
if !strings.Contains(low, "kapcsolatot") {
|
||||
t.Errorf("the sentence must send the household to support: %q", s)
|
||||
}
|
||||
}
|
||||
_, en := m.RestoreHoldForLang("immich", "en")
|
||||
if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") {
|
||||
t.Errorf("English hold sentence: %q", en)
|
||||
}
|
||||
if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") {
|
||||
t.Errorf("English error sentence: %q", enErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR893_AVersionChangeThenAFailedReplayHoldsTheAppAtItsLiveDefinition(t *testing.T) {
|
||||
m, vp, rolled, notified := r893Fixture(t, true, 0)
|
||||
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
||||
assertR893Held(t, m, vp, err, *rolled, *notified)
|
||||
// (a)
|
||||
if len(vp.defs) != 2 {
|
||||
t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs)
|
||||
}
|
||||
if got := strings.Join(vp.defs[1], " "); !strings.Contains(got, "postgres:18-alpine") {
|
||||
t.Fatalf("the definition written back is %q — want the live 18", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR893_AReplacedVolumeThenAFailedReplayHoldsTheApp(t *testing.T) {
|
||||
m, vp, rolled, notified := r893Fixture(t, false, 1)
|
||||
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
||||
assertR893Held(t, m, vp, err, *rolled, *notified)
|
||||
if len(vp.defs) != 0 {
|
||||
t.Fatalf("no version changed, yet a definition was written: %v", vp.defs)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user