R-1256: four restore edges (review round 2 HUNT-1 H4/H5/H6/H8) — each test seen failing first
gates / gates (push) Successful in 31s

H4: a failed definition write in the off-site restore writes the live definition back before the start;
    a failed write-back holds the app (restore_mixed).
H5: a put-back whose stop fails pours nothing, keeps the copy and holds the app (off-site, local, power-cut);
    a failed definition write-back keeps the copy (D20); a failed 'volume rm -f' fails that volume (no merge).
H6: TryBeginRestoreOp/RestoreBusy: check every operation and record the restore in one step under m.mu;
    every web handler and the kept loads begin through it; install-time kept load asks about the mirror too.
H8: SweepUnusedRestoreCopies every 6 h: removes a copy only with no hold, no marker naming it, no restore
    running, and older than 14 days.

Commit only; rides the next controller release.
This commit is contained in:
admin committed 2026-10-28 22:24:16 +01:00
1 parent ed3927f03a
commit 342a394bff
20 files changed
+860 -50

No files matched your search

@@ -226,8 +226,13 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
m.holdAppAfterMixedRestore(stack, cause, live)
return util.MsgError(holdKey, stack)
}
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
// Nothing is poured; the copy is kept and the app held (D20: a put-back that fails holds and keeps its copy).
if err := m.stackProvider.StopStack(stack); err != nil {
m.logger.Printf("[WARN] [offbox] %s: stopping the app before the put-back failed: %v", stack, err)
c.keepIfAny()
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
return util.MsgError(holdKey, stack)
}
if volumesMoved {
if err := m.putBack(stack, c); err != nil {
@@ -239,7 +244,8 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
}
if live != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — holding the app", stack, err)
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
return util.MsgError(holdKey, stack)
}
@@ -249,12 +255,19 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
m.restorePutBackNotify(stack, cause)
}
if err := restart(); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: start after the put-back failed: %v (the copy stays at %s)", stack, err, c.dirOrNone())
m.logger.Printf("[ERROR] [offbox] %s: start after the put-back failed: %v (the live volumes and definition are back; the copy %s is removed)", stack, err, c.dirOrNone())
return util.MsgError("err.backup.restore_failed_put_back_not_started", stack)
}
return putBackMessage(stack, filesPlaced)
}
// keepIfAny marks the copy as kept for support (no copy: nothing to keep).
func (c *liveVolumeCopy) keepIfAny() {
if c != nil {
c.keep = true
}
}
func (c *liveVolumeCopy) dirOrNone() string {
if c == nil {
return "(no volume copy)"