R-893 option A (operator D20): a failed off-site restore puts the app's live volumes and definition back exactly and starts it; fit check before the stop; the held sentence made true (R-1169)
gates / gates (push) Successful in 1m11s

This commit is contained in:
2026-10-20 09:17:00 +02:00
parent ae2bfb2f7a
commit 93fb251ea9
13 changed files with 752 additions and 55 deletions
@@ -796,6 +796,27 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
return res, util.MsgError("err.backup.adatbazis_masolat_csonka_nem_indult", stack)
}
// --- R-893 option A (D20 = A): THE LIVE VOLUMES' COPY MUST FIT, BEFORE ANYTHING STOPS -----------
// Every volume the snapshot will replace is copied right after the stop, so a failed restore can put it back
// (offbox_restore_undo.go). Measured here, while nothing is touched: the copy sits on the same namespace as the
// restore's own files, so it is added to the R-357 need and checked by the same exact rule (no multiplier — the
// sizes are measured, not predicted). An unmeasurable volume refuses, as an unmeasurable scratch does.
undoVols := snapshotVolumeNames(filepath.Join(scratchUnit, "volume-dumps"))
var undoExists map[string]bool
if len(undoVols) > 0 {
volNeed, ex, pErr := m.planLiveVolumeCopy(undoVols)
if pErr != nil {
m.logger.Printf("[ERROR] [offbox] %s: REFUSING the destructive restore — the live volumes could not be measured for their copy: %v; the app was NOT stopped", stack, pErr)
return res, util.MsgError("err.backup.restore_undo_unmeasured", stack)
}
if free < need+volNeed {
m.logger.Printf("[WARN] [offbox] %s: REFUSING the destructive restore — need %d B (restore %d B + copy of the live volumes %d B), free %d B on %s; the app was NOT stopped",
stack, need+volNeed, need, volNeed, free, liveNs)
return res, util.MsgError("err.backup.restore_undo_no_space", humanizeBytes(need+volNeed), humanizeBytes(free))
}
undoExists = ex
}
// --- WHICH VERSION COMES BACK (v0.275.0, R-696, `07` §6.6) ---------------------------------
// Until v0.275.0 this path never wrote the definition, so after any update the snapshot's data
// (last night's version) was started by the app's NEW definition — for an engine step that is the
@@ -938,6 +959,22 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if err := m.stackProvider.StopStack(stack); err != nil {
m.logger.Printf("[WARN] [offbox] could not stop %s before reconstitution: %v (continuing)", stack, err)
}
// R-893 option A: the live volumes the snapshot will replace, copied with the app stopped and before anything
// moved. A copy that fails restarts the app as it was — nothing has been touched. Removed when the restore
// ends, unless a failed put-back keeps it for support.
var undo *liveVolumeCopy
if len(undoVols) > 0 {
c, cErr := m.copyLiveVolumes(stack, liveNs, undoVols, undoExists)
if cErr != nil {
m.logger.Printf("[ERROR] [offbox] %s: the copy of the live volumes failed: %v — nothing was restored, starting the app as it was", stack, cErr)
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: restart after a failed volume copy also failed: %v", stack, sErr)
}
return res, util.MsgError("err.backup.restore_undo_copy_failed", stack)
}
undo = c
defer undo.drop(m, stack)
}
// v0.275.0: the snapshot's own definition, written while nothing of the app's data has been touched
// yet — a failure here restarts the app as it was.
if defineFromSnapshot {
@@ -979,9 +1016,10 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if cErr != nil {
// R-893 option C (security review 2026-10-08): once the snapshot's (possibly older) definition
// is written, a start would run that version on the live data — HOLD the app instead.
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
if definitionMoved {
m.holdAppAfterMixedRestore(stack, cErr, liveDef)
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
}
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1013,9 +1051,10 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// already be the snapshot's — HOLD the app rather than start it on a mix.
// R-893 option C (security review 2026-10-08): a volume already replaced, or the snapshot's
// definition written → HOLD (the volume detail reaches the operator through the hold notice).
if definitionMoved || nVols > 0 {
m.holdAppAfterMixedRestore(stack, vErr, liveDef)
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
if definitionMoved || nVols > 0 || undo != nil {
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
}
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
if sErr := restartStack(); sErr != nil {
@@ -1034,8 +1073,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
if definitionMoved || res.VolumesReplayed > 0 {
m.holdAppAfterMixedRestore(stack, err, liveDef)
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
}
// Best-effort bring-up: a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1065,6 +1103,9 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// the customer type into it and makes the damage permanent. Hold it instead —
// operator ruling, 2026-08-22.
m.logger.Printf("[ERROR] [offbox] %s: ROLLBACK ALSO FAILED (%v) — holding the app stopped; replay error was: %v", stack, rbErr, iErr)
if undo != nil {
undo.keep = true // R-893: the volumes' copy is a second way back for support
}
m.holdAppAfterFailedRollback(stack, iErr, rbErr)
// R-383: the undo copy is DESCRIBED FROM DISK, never from the path alone. See
// undoCopyPhrase — this sentence used to assert the file existed in exactly the
@@ -1080,16 +1121,22 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA.
// Placed FILES alone do not hold (option C as ruled): that mix is what „put back exactly as it
// was" (option A, the next slice) removes.
// R-893 option A (D20 = A): the rows are back, but the snapshot's volumes (and maybe its definition) are
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
if definitionMoved || res.VolumesReplayed > 0 {
m.holdAppAfterMixedRestore(stack, iErr, liveDef)
return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack)
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
}
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
}
// Says BOTH things. A message that reported only the failure would leave the customer
// believing their data was gone when it is back — the omission of a GAIN is as
// misleading as the omission of a loss.
// misleading as the omission of a loss. R-893: placed files are not taken back, so with
// placed files the sentence says that rather than „your data is back as it was".
if res.FilesPlaced > 0 {
return res, putBackMessage(stack, res.FilesPlaced)
}
return res, util.MsgError("err.backup.db_restore_failed_rolled_back", stack)
}
}