diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index f889487..2867c89 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -1371,6 +1371,13 @@ func main() { // path over. backupMgr.SetRestoreHoldNotify(func(stack string, replayErr, rollbackErr error) { d := notify.BackupRunFailuresDetails{RunKind: "offsite-reconstitute", Failed: 1, Attempted: 1} + if rollbackErr == nil { // R-893: the rollback WORKED, but files/volumes/version had already moved + d.Apps = append(d.Apps, notify.RunFailureDetail{App: stack, Leg: "restore-hold-mixed", Reason: "replay failed after files/volumes/version moved"}) + msg := fmt.Sprintf("App %q is HELD STOPPED for support: its off-site database restore failed. The database was rolled back to the pre-restore copy and the live definition written back, "+ + "but its files and named volumes are the snapshot's — a mixed state (R-893). The app will not start from any path until the hold is cleared. Replay error: %v", stack, replayErr) + notifier.NotifyBackupRunFailures(msg, d) + return + } reason := "rollback failed" if rollbackErr != nil { reason = rollbackErr.Error() diff --git a/controller/internal/backup/offbox_reconstitute.go b/controller/internal/backup/offbox_reconstitute.go index f2de636..f456c0c 100644 --- a/controller/internal/backup/offbox_reconstitute.go +++ b/controller/internal/backup/offbox_reconstitute.go @@ -369,6 +369,9 @@ func (m *Manager) RestoreHoldForLang(stack, lang string) (bool, string) { if t, err := time.Parse(time.RFC3339, h.At); err == nil { when = t.Format("2006-01-02 15:04") } + if h.Reason == settings.HoldReasonRestoreMixed { // R-893 + return true, util.Text(lang, "note.reconstitute.held_mixed", stack, when) + } return true, util.Text(lang, "note.reconstitute.held", stack, when) } @@ -441,6 +444,93 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr } } +// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's +// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only. +type liveDefinition struct { + dir string + env map[string]string +} + +func (d *liveDefinition) cleanup() { + if d != nil && d.dir != "" { + _ = os.RemoveAll(d.dir) + } +} + +// captureLiveDefinition copies the live docker-compose.yml (required) and .felhom.yml (when present) and +// reads the live env (non-secret values plus the live secrets, decrypted from the guest's app.yaml) — the +// same two sources the restore path uses. Values are never logged. +func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) { + composePath, ok := m.stackProvider.GetStackComposePath(stack) + if !ok || composePath == "" { + return nil, fmt.Errorf("no live compose path for %s", stack) + } + dir, err := os.MkdirTemp("", "felhom-livedef-") + if err != nil { + return nil, err + } + d := &liveDefinition{dir: dir, env: map[string]string{}} + for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} { + data, rErr := os.ReadFile(filepath.Join(filepath.Dir(composePath), fname)) + if rErr != nil { + if fname == "docker-compose.yml" { + d.cleanup() + return nil, rErr + } + continue + } + if wErr := os.WriteFile(filepath.Join(dir, fname), data, 0o600); wErr != nil { + d.cleanup() + return nil, wErr + } + } + if info, ok := m.stackProvider.GetStackRecoveryInfo(stack); ok { + for k, v := range info.NonSecretEnv { + d.env[k] = v + } + for k, v := range m.stackProvider.RecoverStackSecrets(stack, info.SecretEnvVars) { + d.env[k] = v + } + } + return d, nil +} + +// holdAppAfterMixedRestore (R-893, `09` §3 decision 192, option C) is the hold for a failed replay whose +// database rollback WORKED but which had already moved more than the database: the live definition is +// written back (when the snapshot's was written), the database service is stopped again, and the app is +// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the +// notification tells this case from R-379's double failure. +func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) { + if live != nil { + if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil { + m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the app is held at the snapshot's definition", stack, err) + } else { + m.logger.Printf("[INFO] [offbox] %s: the live definition is written back", stack) + } + } + if err := m.stackProvider.StopStack(stack); err != nil { + m.logger.Printf("[WARN] [offbox] %s: stopping the database service before the hold failed: %v", stack, err) + } + m.logger.Printf("[ERROR] [offbox] %s: database rolled back, but files/volumes/version had already moved — HOLDING the app stopped for support (R-893); replay error was: %v", stack, replayErr) + if m.settings == nil { + m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack) + return + } + h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed} + if replayErr != nil { + h.ReplayError = replayErr.Error() + } + if err := m.settings.SetRestoreHold(h); err != nil { + m.logger.Printf("[ERROR] [offbox] %s: persisting the restore hold FAILED: %v — the app is stopped and unguarded", stack, err) + } + if m.appStop != nil { + m.appStop.End() + } + if m.restoreHoldNotify != nil { + m.restoreHoldNotify(stack, replayErr, nil) + } +} + // SetRestoreHoldNotify wires the operator notification for a held app (cmd/controller/main.go). func (m *Manager) SetRestoreHoldNotify(fn func(stack string, replayErr, rollbackErr error)) { m.restoreHoldNotify = fn @@ -732,6 +822,21 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack } } + // --- THE LIVE DEFINITION, KEPT FOR A FAILED REPLAY (R-893, `09` §3 decision 192) ------------- + // When the snapshot's definition is about to be written, keep the LIVE one first, so a failed + // replay below can write it back before the app is held. Taken before the first mutation; a + // failure here refuses with nothing touched — without it the failure branch could not be undone. + var liveDef *liveDefinition + if defineFromSnapshot { + ld, ldErr := m.captureLiveDefinition(stack) + if ldErr != nil { + m.logger.Printf("[ERROR] [offbox] Restore REFUSED for %s: could not keep the live definition: %v — nothing was touched", stack, ldErr) + return res, fmt.Errorf("restoring %s: keeping the live definition failed: %w", stack, ldErr) + } + liveDef = ld + defer liveDef.cleanup() + } + // --- WHICH SERVICE HOLDS THE DATABASE (R-47) ------------------------------------------------ // Read from the compose that will RUN: the live one, or — when the app comes back at the snapshot's // version — the snapshot unit's, which is written into the stack dir before the first start. Resolved @@ -907,6 +1012,16 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack return res, util.MsgError("err.backup.db_restore_and_rollback_failed", stack, m.undoCopyPhrase(safetySet)) } res.RolledBack = true + // --- R-893: A ROLLED-BACK DATABASE IS NOT „BACK AS IT WAS" WHEN MORE MOVED --------- + // The rollback puts back the database rows only. When the snapshot's definition was + // written or a named volume was replaced, the app would start on a mix: the snapshot's + // files and volumes (and maybe its older version) under the newer database. Write the + // live definition back, stop the database service again, and HOLD the app for support. + // Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA. + if defineFromSnapshot || res.VolumesReplayed > 0 { + m.holdAppAfterMixedRestore(stack, iErr, liveDef) + return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack) + } if sErr := restartStack(); sErr != nil { m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr) } diff --git a/controller/internal/backup/r893_mixed_restore_test.go b/controller/internal/backup/r893_mixed_restore_test.go new file mode 100644 index 0000000..8d832a7 --- /dev/null +++ b/controller/internal/backup/r893_mixed_restore_test.go @@ -0,0 +1,135 @@ +package backup + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" + "gitea.dooplex.hu/admin/felhom-controller/internal/util" +) + +// R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md). +// +// An off-site restore of one app writes the snapshot's definition (when the version differs), copies the +// snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails, +// the rollback puts back only the database rows. Until this slice the app was started on that mix — +// an older definition, the snapshot's files and volumes, the newer database — and the household was told +// „your data is back as it was before the restore". That is true only of the database rows. +// +// Now: the database is still rolled back, the LIVE definition is written back, and the app is HELD +// stopped with a sentence that says it needs support. Pinned here: +// (a) the definition write is called a second time, with the live definition; +// (b) the app is NOT started; +// (c) a hold is left (kind restore_mixed), and the operator is notified; +// (d) neither the error nor the hold sentence claims the data is back as it was (hu and en). +// The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA. + +// r893Provider records EVERY definition write (vtReconProvider keeps only the last). +type r893Provider struct { + *vtReconProvider + defs [][]string +} + +func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error { + p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml"))) + return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env) +} + +func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) { + t.Helper() + m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1)) + m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") } + rolled := 0 + m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil }) + notified := 0 + m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) { + notified++ + if replayErr == nil || rollbackErr != nil { + t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr) + } + }) + m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil } + live := "16" + if versionChange { + live = "18" + } + // The LIVE definition, on disk where the app runs. + if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil { + t.Fatal(err) + } + vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}} + m.SetStackProvider(vp) + vtSnapshotUnit(t, m, "16", true) + return m, vp, &rolled, ¬ified +} + +func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) { + t.Helper() + if err == nil { + t.Fatal("a failed replay must be surfaced as a failure") + } + if rolled != 1 { + t.Fatalf("the database rollback must still run exactly once, ran %d", rolled) + } + // (b) + if vp.fullStarted { + t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls) + } + if last := vp.calls[len(vp.calls)-1]; last != "stop" { + t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls) + } + // (c) + held, sentence := m.RestoreHoldFor("immich") + if !held { + t.Fatal("no hold was left — every start path would start the app on the mixed state") + } + if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed { + t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed) + } + if notified != 1 { + t.Errorf("the operator must be notified once, got %d", notified) + } + // (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too). + for _, s := range []string{err.Error(), sentence} { + low := strings.ToLower(s) + if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") { + t.Errorf("the sentence still claims the data is back / the app runs: %q", s) + } + if !strings.Contains(low, "kapcsolatot") { + t.Errorf("the sentence must send the household to support: %q", s) + } + } + _, en := m.RestoreHoldForLang("immich", "en") + if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") { + t.Errorf("English hold sentence: %q", en) + } + if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") { + t.Errorf("English error sentence: %q", enErr) + } +} + +func TestR893_AVersionChangeThenAFailedReplayHoldsTheAppAtItsLiveDefinition(t *testing.T) { + m, vp, rolled, notified := r893Fixture(t, true, 0) + _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) + assertR893Held(t, m, vp, err, *rolled, *notified) + // (a) + if len(vp.defs) != 2 { + t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs) + } + if got := strings.Join(vp.defs[1], " "); !strings.Contains(got, "postgres:18-alpine") { + t.Fatalf("the definition written back is %q — want the live 18", got) + } +} + +func TestR893_AReplacedVolumeThenAFailedReplayHoldsTheApp(t *testing.T) { + m, vp, rolled, notified := r893Fixture(t, false, 1) + _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false) + assertR893Held(t, m, vp, err, *rolled, *notified) + if len(vp.defs) != 0 { + t.Fatalf("no version changed, yet a definition was written: %v", vp.defs) + } +} diff --git a/controller/internal/i18n/locales/en.json b/controller/internal/i18n/locales/en.json index 19499c8..15dd291 100644 --- a/controller/internal/i18n/locales/en.json +++ b/controller/internal/i18n/locales/en.json @@ -1264,6 +1264,7 @@ "err.backup.az_offsite_tarolo_nincs_elarvult_allapotban": "the off-site store is not orphaned", "err.backup.db_restore_and_rollback_failed": "putting back the database of %s failed, and the earlier state could not be restored either. We have left the app STOPPED, for safety, so your data cannot be damaged further. Get in touch with us — %s", "err.backup.db_restore_failed_rolled_back": "putting back the database of %s failed — your data is back as it was before the restore, and the app is still running. If you want to try again, get in touch with us first", + "err.backup.db_restore_failed_held_mixed": "putting back the database of %s failed. The database is as it was before the restore, but the files and other data of the app already come from the backup, so they do not match. We have left the app STOPPED, for safety. This needs our help: get in touch with us", "err.backup.egy_masik_mentesi_visszaallitasi_muvelet_mar": "another backup or restore is already running", "err.backup.ennek_az_alkalmazasnak_az_adatai_nem": "this app’s data cannot be restored from this copy", "err.backup.ervenytelen_alkalmazasnev": "that app name is not valid", @@ -1780,6 +1781,7 @@ "note.offsite.whole_unit_gap": "(the whole app — it has no local backup unit)", "note.reconstitute.copy_latest": "latest", "note.reconstitute.held": "restoring the data of %s stopped at %s, and the earlier state could not be put back either. The app stays stopped, for safety, so your data cannot be damaged further. Get in touch with us", + "note.reconstitute.held_mixed": "restoring %s stopped half-way at %s: the database is as it was before the restore, the other data comes from the backup. The app stays stopped, for safety. This needs our help: get in touch with us", "note.restore.whole_files": "The drive's files: %d brought back, %d replaced (the older live copy was kept beside it, ending in .felhom-…), %d newer copies left untouched, %d unchanged. No file was deleted.", "note.restore.all_files_present": "Every file that was checked is in place.", "note.restore.and_database": " and the database", diff --git a/controller/internal/i18n/locales/hu.json b/controller/internal/i18n/locales/hu.json index 6093bee..13c2b29 100644 --- a/controller/internal/i18n/locales/hu.json +++ b/controller/internal/i18n/locales/hu.json @@ -1259,6 +1259,7 @@ "err.backup.az_offsite_tarolo_nincs_elarvult_allapotban": "az offsite tároló nincs elárvult állapotban", "err.backup.db_restore_and_rollback_failed": "a(z) %s adatbázisának visszaállítása sikertelen, és a korábbi állapot visszatöltése sem sikerült. Az alkalmazást biztonsági okból LEÁLLÍTVA hagytuk, hogy az adatai ne sérüljenek tovább. Vedd fel velünk a kapcsolatot — %s", "err.backup.db_restore_failed_rolled_back": "a(z) %s adatbázisának visszaállítása sikertelen — az adataid visszakerültek a visszaállítás előtti állapotba, az alkalmazás fut tovább. Ha újra megpróbálnád, előbb vedd fel velünk a kapcsolatot", + "err.backup.db_restore_failed_held_mixed": "a(z) %s adatbázisának visszaállítása sikertelen. Az adatbázis a visszaállítás előtti állapotban van, de az alkalmazás fájljai és többi adata már a mentésből származnak, ezért nem illenek össze. Az alkalmazást biztonsági okból LEÁLLÍTVA hagytuk. Ehhez segítség kell: vedd fel velünk a kapcsolatot", "err.backup.egy_masik_mentesi_visszaallitasi_muvelet_mar": "egy másik mentési/visszaállítási művelet már fut", "err.backup.ennek_az_alkalmazasnak_az_adatai_nem": "ennek az alkalmazásnak az adatai nem ebből a másolatból állíthatók vissza", "err.backup.ervenytelen_alkalmazasnev": "érvénytelen alkalmazásnév", @@ -1768,6 +1769,7 @@ "note.offsite.whole_unit_gap": "(a teljes alkalmazás — nincs helyi mentési egysége)", "note.reconstitute.copy_latest": "legutóbbi", "note.reconstitute.held": "a(z) %s adatainak visszaállítása %s-kor megszakadt, és a korábbi állapotot sem sikerült visszatölteni. Az alkalmazás biztonsági okból leállítva marad, hogy az adatai ne sérüljenek tovább. Vedd fel velünk a kapcsolatot", + "note.reconstitute.held_mixed": "a(z) %s visszaállítása %s-kor félúton megszakadt: az adatbázis a visszaállítás előtti állapotban van, a többi adat a mentésből származik. Az alkalmazás biztonsági okból leállítva marad. Ehhez segítség kell: vedd fel velünk a kapcsolatot", "note.restore.whole_files": "A meghajtó fájljai: %d visszahozva, %d kicserélve (a régebbi élő példány megmaradt mellette, .felhom-… végződéssel), %d újabb példány érintetlenül hagyva, %d változatlan. Fájl nem lett törölve.", "note.restore.all_files_present": "Minden vizsgált fájl megvan a helyén.", "note.restore.and_database": " és az adatbázis", diff --git a/controller/internal/settings/settings.go b/controller/internal/settings/settings.go index 0e7babe..5a0df2d 100644 --- a/controller/internal/settings/settings.go +++ b/controller/internal/settings/settings.go @@ -1876,6 +1876,11 @@ const ( // HoldReasonUnhealthyStop (v0.269.0, `09` §3 decision 28): the box stopped an app in a crash loop or // an out-of-memory storm. Lifted by the household's Start (one more try); nothing else starts it. HoldReasonUnhealthyStop = "unhealthy_stop" + // HoldReasonRestoreMixed (R-893, `09` §3 decision 192): an off-site restore of one app failed its database + // replay AFTER the snapshot's definition was written or a named volume was replaced. The database was + // rolled back, the live definition written back, and the app is held: its files and volumes are the + // snapshot's, its database the pre-restore one. Cleared like a restore hold — by the operator. + HoldReasonRestoreMixed = "restore_mixed" ) // SetRestoreHold records a hold. Modelled on SetDisconnected: a condition, plus what it is holding. diff --git a/controller/scripts/i18n_go_keys.json b/controller/scripts/i18n_go_keys.json index af743e4..6a6d90a 100644 --- a/controller/scripts/i18n_go_keys.json +++ b/controller/scripts/i18n_go_keys.json @@ -1,6 +1,8 @@ { "_comment": "Localisation slice 2 (R-557). key -> the base-commit Go literal it replaced, or the ORDERED list of literals a concatenation joined. Checked by scripts/i18n_go_parity.py against scripts/i18n_go_base.json, which is frozen at 736f54b49610 (the base commit of release v0.252.0). A key whose Hungarian text is not byte-identical to what the Go code said fails the gate.", "_preexisting": { + "err.backup.db_restore_failed_held_mixed": "BORN AS A KEY (R-893, 2026-10-08) -- a NEW sentence for a failed off-site replay whose database rollback worked but whose files/volumes/version had already moved; the app is held. Never a Go literal; pinned by internal/backup/r893_mixed_restore_test.go.", + "note.reconstitute.held_mixed": "BORN AS A KEY (R-893, 2026-10-08) -- the hold sentence for that held app (HoldReasonRestoreMixed). Never a Go literal; pinned by internal/backup/r893_mixed_restore_test.go.", "recovery.older_unchecked": "BORN AS A KEY (R-304, 2026-10-08) -- a NEW sentence of the recovery screen for the agent's 424 (earlier sealed packages not all checked), never a Go literal; pinned by TestR304_OlderUnchecked_IsNotAWrongCode.", "banner.missed_backup.never": "BORN AS A KEY, v0.295.0 (R-871, `09` decision 110) -- a NEW sentence of the missed-backup banner, never a Go literal; pinned in Hungarian by TestR871_BannerShownThenClosedUntilTheNextMiss and the parity fixture launcher_missed_backup.", "banner.missed_backup.last": "BORN AS A KEY, v0.295.0 (R-871, `09` decision 110) -- a NEW sentence of the missed-backup banner, never a Go literal; pinned in Hungarian by TestR871_BannerShownThenClosedUntilTheNextMiss and the parity fixture launcher_missed_backup.",