clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s

- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING
  controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back
  from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type.
- --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid
  checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart).
- R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause,
  failed put-back step, database replay or not, kept copy) — never the option-C database story by default.
- R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
admin committed 2026-10-29 12:49:46 +01:00
1 parent c7bb776e00
commit 5b3b541602
25 files changed
+1809 -83

No files matched your search

@@ -11,6 +11,7 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
//
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
// the household's sentence), so the operator's mail names the real cause.
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
filesPlaced int, restart func() error, holdKey string) error {
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
why := func(step string) mixedHoldWhy {
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
if c != nil && c.keep {
w.copyDir = c.dir
}
return w
}
if volumesMoved && c == nil {
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
return util.MsgError(holdKey, stack)
}
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.StopStack(stack); err != nil {
c.keepIfAny()
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
return util.MsgError(holdKey, stack)
}
if volumesMoved {
if err := m.putBack(stack, c); err != nil {
c.keep = true
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}