R-1256: four restore edges (review round 2 HUNT-1 H4/H5/H6/H8) — each test seen failing first
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
H4: a failed definition write in the off-site restore writes the live definition back before the start;
a failed write-back holds the app (restore_mixed).
H5: a put-back whose stop fails pours nothing, keeps the copy and holds the app (off-site, local, power-cut);
a failed definition write-back keeps the copy (D20); a failed 'volume rm -f' fails that volume (no merge).
H6: TryBeginRestoreOp/RestoreBusy: check every operation and record the restore in one step under m.mu;
every web handler and the kept loads begin through it; install-time kept load asks about the mirror too.
H8: SweepUnusedRestoreCopies every 6 h: removes a copy only with no hold, no marker naming it, no restore
running, and older than 14 days.
Commit only; rides the next controller release.
This commit is contained in:
1 parent
ed3927f03a
commit
342a394bff
20 files changed
+860
-50
No files matched your search
@@ -548,6 +548,25 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
|
||||
}
|
||||
}
|
||||
|
||||
// restartAtLiveDefinition (R-1256 H4) is the failure branch of the definition write, taken while no data has moved.
|
||||
// The writer writes the compose before the app's settings, so a failure can leave the snapshot's compose in the
|
||||
// stack dir: the LIVE definition is written back first, and only then is the app started. When the write-back fails
|
||||
// too, the app is held stopped (restore_mixed) — a start would run the snapshot's definition over the live data.
|
||||
// Pinned by TestR1256_H4_*.
|
||||
func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, cause error, restartStack func() error) {
|
||||
if live != nil {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
|
||||
m.holdAppAfterMixedRestore(stack, cause, nil)
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
|
||||
}
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: restart after a failed definition write also failed: %v", stack, sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// SetRestoreHoldNotify wires the operator notification for a held app (cmd/controller/main.go).
|
||||
func (m *Manager) SetRestoreHoldNotify(fn func(stack string, replayErr, rollbackErr error)) {
|
||||
m.restoreHoldNotify = fn
|
||||
@@ -1005,16 +1024,12 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// yet — a failure here restarts the app as it was.
|
||||
if defineFromSnapshot {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, scratchCompose, snapEnv); err != nil {
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: restart after a failed definition write also failed: %v", stack, sErr)
|
||||
}
|
||||
m.restartAtLiveDefinition(stack, liveDef, err, restartStack)
|
||||
return res, fmt.Errorf("restoring %s: writing the snapshot's definition failed: %w", stack, err)
|
||||
}
|
||||
} else if len(credOverlay) > 0 {
|
||||
if err := m.stackProvider.(secretValueSetter).SetStackSecretValues(stack, credOverlay); err != nil {
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: restart after a failed password write also failed: %v", stack, sErr)
|
||||
}
|
||||
m.restartAtLiveDefinition(stack, liveDef, err, restartStack)
|
||||
return res, fmt.Errorf("restoring %s: writing the backup's database password failed: %w", stack, err)
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] %s: the backup's database password(s) %v written into the app's settings before the data (R-1106)", stack, credNames(credOverlay))
|
||||
|
||||
Reference in new issue
Block a user