clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type. - --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart). - R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause, failed put-back step, database replay or not, kept copy) — never the option-C database story by default. - R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
1 parent
c7bb776e00
commit
5b3b541602
25 files changed
+1809
-83
No files matched your search
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
|
||||
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
|
||||
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
|
||||
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
|
||||
//
|
||||
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
|
||||
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
|
||||
// the household's sentence), so the operator's mail names the real cause.
|
||||
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
|
||||
filesPlaced int, restart func() error, holdKey string) error {
|
||||
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
|
||||
why := func(step string) mixedHoldWhy {
|
||||
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
|
||||
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
|
||||
if c != nil && c.keep {
|
||||
w.copyDir = c.dir
|
||||
}
|
||||
return w
|
||||
}
|
||||
if volumesMoved && c == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
|
||||
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.StopStack(stack); err != nil {
|
||||
c.keepIfAny()
|
||||
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
if volumesMoved {
|
||||
if err := m.putBack(stack, c); err != nil {
|
||||
c.keep = true
|
||||
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
|
||||
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user