R-893 option A (operator D20): a failed off-site restore puts the app's live volumes and definition back exactly and starts it; fit check before the stop; the held sentence made true (R-1169)
gates / gates (push) Successful in 1m11s
gates / gates (push) Successful in 1m11s
This commit is contained in:
@@ -796,6 +796,27 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
return res, util.MsgError("err.backup.adatbazis_masolat_csonka_nem_indult", stack)
|
||||
}
|
||||
|
||||
// --- R-893 option A (D20 = A): THE LIVE VOLUMES' COPY MUST FIT, BEFORE ANYTHING STOPS -----------
|
||||
// Every volume the snapshot will replace is copied right after the stop, so a failed restore can put it back
|
||||
// (offbox_restore_undo.go). Measured here, while nothing is touched: the copy sits on the same namespace as the
|
||||
// restore's own files, so it is added to the R-357 need and checked by the same exact rule (no multiplier — the
|
||||
// sizes are measured, not predicted). An unmeasurable volume refuses, as an unmeasurable scratch does.
|
||||
undoVols := snapshotVolumeNames(filepath.Join(scratchUnit, "volume-dumps"))
|
||||
var undoExists map[string]bool
|
||||
if len(undoVols) > 0 {
|
||||
volNeed, ex, pErr := m.planLiveVolumeCopy(undoVols)
|
||||
if pErr != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: REFUSING the destructive restore — the live volumes could not be measured for their copy: %v; the app was NOT stopped", stack, pErr)
|
||||
return res, util.MsgError("err.backup.restore_undo_unmeasured", stack)
|
||||
}
|
||||
if free < need+volNeed {
|
||||
m.logger.Printf("[WARN] [offbox] %s: REFUSING the destructive restore — need %d B (restore %d B + copy of the live volumes %d B), free %d B on %s; the app was NOT stopped",
|
||||
stack, need+volNeed, need, volNeed, free, liveNs)
|
||||
return res, util.MsgError("err.backup.restore_undo_no_space", humanizeBytes(need+volNeed), humanizeBytes(free))
|
||||
}
|
||||
undoExists = ex
|
||||
}
|
||||
|
||||
// --- WHICH VERSION COMES BACK (v0.275.0, R-696, `07` §6.6) ---------------------------------
|
||||
// Until v0.275.0 this path never wrote the definition, so after any update the snapshot's data
|
||||
// (last night's version) was started by the app's NEW definition — for an engine step that is the
|
||||
@@ -938,6 +959,22 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
if err := m.stackProvider.StopStack(stack); err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] could not stop %s before reconstitution: %v (continuing)", stack, err)
|
||||
}
|
||||
// R-893 option A: the live volumes the snapshot will replace, copied with the app stopped and before anything
|
||||
// moved. A copy that fails restarts the app as it was — nothing has been touched. Removed when the restore
|
||||
// ends, unless a failed put-back keeps it for support.
|
||||
var undo *liveVolumeCopy
|
||||
if len(undoVols) > 0 {
|
||||
c, cErr := m.copyLiveVolumes(stack, liveNs, undoVols, undoExists)
|
||||
if cErr != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: the copy of the live volumes failed: %v — nothing was restored, starting the app as it was", stack, cErr)
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: restart after a failed volume copy also failed: %v", stack, sErr)
|
||||
}
|
||||
return res, util.MsgError("err.backup.restore_undo_copy_failed", stack)
|
||||
}
|
||||
undo = c
|
||||
defer undo.drop(m, stack)
|
||||
}
|
||||
// v0.275.0: the snapshot's own definition, written while nothing of the app's data has been touched
|
||||
// yet — a failure here restarts the app as it was.
|
||||
if defineFromSnapshot {
|
||||
@@ -979,9 +1016,10 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
if cErr != nil {
|
||||
// R-893 option C (security review 2026-10-08): once the snapshot's (possibly older) definition
|
||||
// is written, a start would run that version on the live data — HOLD the app instead.
|
||||
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
|
||||
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
|
||||
if definitionMoved {
|
||||
m.holdAppAfterMixedRestore(stack, cErr, liveDef)
|
||||
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
|
||||
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
}
|
||||
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1013,9 +1051,10 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// already be the snapshot's — HOLD the app rather than start it on a mix.
|
||||
// R-893 option C (security review 2026-10-08): a volume already replaced, or the snapshot's
|
||||
// definition written → HOLD (the volume detail reaches the operator through the hold notice).
|
||||
if definitionMoved || nVols > 0 {
|
||||
m.holdAppAfterMixedRestore(stack, vErr, liveDef)
|
||||
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
|
||||
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
|
||||
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
|
||||
if definitionMoved || nVols > 0 || undo != nil {
|
||||
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
}
|
||||
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1034,8 +1073,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
|
||||
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
m.holdAppAfterMixedRestore(stack, err, liveDef)
|
||||
return res, util.MsgError("err.backup.restore_failed_held_mixed", stack)
|
||||
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
}
|
||||
// Best-effort bring-up: a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1065,6 +1103,9 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// the customer type into it and makes the damage permanent. Hold it instead —
|
||||
// operator ruling, 2026-08-22.
|
||||
m.logger.Printf("[ERROR] [offbox] %s: ROLLBACK ALSO FAILED (%v) — holding the app stopped; replay error was: %v", stack, rbErr, iErr)
|
||||
if undo != nil {
|
||||
undo.keep = true // R-893: the volumes' copy is a second way back for support
|
||||
}
|
||||
m.holdAppAfterFailedRollback(stack, iErr, rbErr)
|
||||
// R-383: the undo copy is DESCRIBED FROM DISK, never from the path alone. See
|
||||
// undoCopyPhrase — this sentence used to assert the file existed in exactly the
|
||||
@@ -1080,16 +1121,22 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// Pinned by r893_mixed_restore_test.go; the plain case keeps TestR379_ScenarioA.
|
||||
// Placed FILES alone do not hold (option C as ruled): that mix is what „put back exactly as it
|
||||
// was" (option A, the next slice) removes.
|
||||
// R-893 option A (D20 = A): the rows are back, but the snapshot's volumes (and maybe its definition) are
|
||||
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
|
||||
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
m.holdAppAfterMixedRestore(stack, iErr, liveDef)
|
||||
return res, util.MsgError("err.backup.db_restore_failed_held_mixed", stack)
|
||||
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
|
||||
}
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
|
||||
}
|
||||
// Says BOTH things. A message that reported only the failure would leave the customer
|
||||
// believing their data was gone when it is back — the omission of a GAIN is as
|
||||
// misleading as the omission of a loss.
|
||||
// misleading as the omission of a loss. R-893: placed files are not taken back, so with
|
||||
// placed files the sentence says that rather than „your data is back as it was".
|
||||
if res.FilesPlaced > 0 {
|
||||
return res, putBackMessage(stack, res.FilesPlaced)
|
||||
}
|
||||
return res, util.MsgError("err.backup.db_restore_failed_rolled_back", stack)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user