R-1256: four restore edges (review round 2 HUNT-1 H4/H5/H6/H8) — each test seen failing first
gates / gates (push) Successful in 31s

H4: a failed definition write in the off-site restore writes the live definition back before the start;
    a failed write-back holds the app (restore_mixed).
H5: a put-back whose stop fails pours nothing, keeps the copy and holds the app (off-site, local, power-cut);
    a failed definition write-back keeps the copy (D20); a failed 'volume rm -f' fails that volume (no merge).
H6: TryBeginRestoreOp/RestoreBusy: check every operation and record the restore in one step under m.mu;
    every web handler and the kept loads begin through it; install-time kept load asks about the mirror too.
H8: SweepUnusedRestoreCopies every 6 h: removes a copy only with no hold, no marker naming it, no restore
    running, and older than 14 days.

Commit only; rides the next controller release.
This commit is contained in:
admin committed 2026-10-28 22:24:16 +01:00
1 parent ed3927f03a
commit 342a394bff
20 files changed
+860 -50

No files matched your search

@@ -548,6 +548,25 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
}
}
// restartAtLiveDefinition (R-1256 H4) is the failure branch of the definition write, taken while no data has moved.
// The writer writes the compose before the app's settings, so a failure can leave the snapshot's compose in the
// stack dir: the LIVE definition is written back first, and only then is the app started. When the write-back fails
// too, the app is held stopped (restore_mixed) — a start would run the snapshot's definition over the live data.
// Pinned by TestR1256_H4_*.
func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, cause error, restartStack func() error) {
if live != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
m.holdAppAfterMixedRestore(stack, cause, nil)
return
}
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
}
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: restart after a failed definition write also failed: %v", stack, sErr)
}
}
// SetRestoreHoldNotify wires the operator notification for a held app (cmd/controller/main.go).
func (m *Manager) SetRestoreHoldNotify(fn func(stack string, replayErr, rollbackErr error)) {
m.restoreHoldNotify = fn
@@ -1005,16 +1024,12 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// yet — a failure here restarts the app as it was.
if defineFromSnapshot {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, scratchCompose, snapEnv); err != nil {
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: restart after a failed definition write also failed: %v", stack, sErr)
}
m.restartAtLiveDefinition(stack, liveDef, err, restartStack)
return res, fmt.Errorf("restoring %s: writing the snapshot's definition failed: %w", stack, err)
}
} else if len(credOverlay) > 0 {
if err := m.stackProvider.(secretValueSetter).SetStackSecretValues(stack, credOverlay); err != nil {
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: restart after a failed password write also failed: %v", stack, sErr)
}
m.restartAtLiveDefinition(stack, liveDef, err, restartStack)
return res, fmt.Errorf("restoring %s: writing the backup's database password failed: %w", stack, err)
}
m.logger.Printf("[INFO] [offbox] %s: the backup's database password(s) %v written into the app's settings before the data (R-1106)", stack, credNames(credOverlay))