clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s

- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING
  controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back
  from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type.
- --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid
  checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart).
- R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause,
  failed put-back step, database replay or not, kept copy) — never the option-C database story by default.
- R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
admin committed 2026-10-29 12:49:46 +01:00
1 parent c7bb776e00
commit 5b3b541602
25 files changed
+1809 -83

No files matched your search

@@ -425,9 +425,12 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
return
}
// R-1340: this branch is only ever an off-site restore's database replay (offbox_reconstitute's rollback).
h := settings.RestoreHold{
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
RestoreKind: settings.RestoreKindOffsite,
DBReplay: true,
}
if replayErr != nil {
h.ReplayError = replayErr.Error()
@@ -447,6 +450,15 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
}
}
// mixedHoldWhy (R-1340) is what a restore_mixed hold records about its cause (settings.RestoreHold's R-1340 fields).
type mixedHoldWhy struct {
kind string // settings.RestoreKind*
cause string // settings.MixedCause*
step string // the step that failed, with its error
dbReplay bool // the restore's own failure was its database replay
copyDir string // where the copy of the live volumes is kept; "" = none kept
}
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
type liveDefinition struct {
@@ -511,7 +523,11 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
// written back (when the snapshot's was written), the database service is stopped again, and the app is
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
// notification tells this case from R-379's double failure.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
//
// why (R-1340) is what the hold records about its cause, so the operator's mail tells the real story — this hold is
// written by three different branches now (a definition write, a put-back that failed at one of its steps), not
// only by option C's database replay.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition, why mixedHoldWhy) {
// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
@@ -519,7 +535,8 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
if m.settings == nil {
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
} else {
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
RestoreKind: why.kind, MixedCause: why.cause, FailedStep: why.step, DBReplay: why.dbReplay, SafetyDump: why.copyDir}
if replayErr != nil {
h.ReplayError = replayErr.Error()
}
@@ -557,7 +574,8 @@ func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, ca
if live != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
m.holdAppAfterMixedRestore(stack, cause, nil)
m.holdAppAfterMixedRestore(stack, cause, nil, mixedHoldWhy{kind: settings.RestoreKindOffsite, cause: settings.MixedCauseDefinitionWrite,
step: "writing the live app definition back failed: " + err.Error()})
return
}
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
@@ -1060,7 +1078,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
if definitionMoved {
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1095,7 +1113,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
if definitionMoved || nVols > 0 || undo != nil {
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
if sErr := restartStack(); sErr != nil {
@@ -1114,7 +1132,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Best-effort bring-up: a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1166,7 +1184,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed", settings.RestoreKindOffsite)
}
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)