clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s

- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING
  controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back
  from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type.
- --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid
  checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart).
- R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause,
  failed put-back step, database replay or not, kept copy) — never the option-C database story by default.
- R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
admin committed 2026-10-29 12:49:46 +01:00
1 parent c7bb776e00
commit 5b3b541602
25 files changed
+1809 -83

No files matched your search

@@ -119,7 +119,8 @@ func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
return false
}
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir,
MixedCause: settings.MixedCauseInterruptedPutBack, FailedStep: "putting the interrupted restore's copy back failed: " + cause.Error()}
if err := g.holds.SetRestoreHold(h); err != nil {
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
return false
+66
View File
@@ -0,0 +1,66 @@
package backup
import (
"errors"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// ── D64 = B (operator ruling 2026-10-29 12:10) + R-1343: the operator clears a hold IN THE RUNNING CONTROLLER ──────
//
// One door for both of the operator's routes — the hub's „Clear hold" (operator action clear_hold) and the box command
// `--clear-restore-hold`, which now asks the running controller over its local socket (cmd/controller/holdctl.go).
//
// WHY IN THIS PROCESS (R-1343, measured 2026-10-29 on Tester 1): the CLI used to run as a SECOND process that edited
// settings.json while the running controller kept the hold in its own in-memory Settings — and that controller's next
// save (after a hub report) wrote the hold back. Clearing through the running Settings removes the hold from memory
// AND saves the file at once, so no later save of any component can bring it back.
// Pinned by TestD64_ClearHoldStaysGoneAfterALaterSave (cmd/controller).
//
// EVERY REASON (D64 = B): the hold's reason does not matter — R-379 now reads „a person — on the box, or the operator
// on the hub". One reason needs a second act: a put-back WAITING for its drive (HoldReasonPutBackPending) is also
// recorded in the app-stop marker, and a controller start with the drive bound would pour that old copy back over
// whatever the app wrote after the clear. So the clear also drops the app from the marker; the copy stays on disk
// (nothing is deleted). It runs under resumeMu, so it can never interleave with a put-back of the same app.
// Pinned by TestD64_ClearPutBackPendingDropsTheMarkerEntry.
// ErrHoldBadStack: the argument is not a valid stack name. ErrNotHeld: the app has no hold. Both are refusals —
// nothing was changed.
var (
ErrHoldBadStack = errors.New("not a valid app name")
ErrNotHeld = errors.New("this app has no hold")
)
// IsValidHoldStack reports whether s may name an app for the hold commands (the backup package's strict
// isSafeStackName: letters, digits, '-' and '_', at most 64).
func IsValidHoldStack(s string) bool { return isSafeStackName(s) }
// ClearHoldByOperator lifts stack's hold, whatever its reason, in the hold store the running controller uses, and
// returns the hold it lifted. A waiting put-back is dropped from the app-stop marker too (see above). Refusals
// (ErrHoldBadStack, ErrNotHeld) change nothing.
func (g *AppStopGuard) ClearHoldByOperator(stack string) (settings.RestoreHold, error) {
if !isSafeStackName(stack) {
return settings.RestoreHold{}, ErrHoldBadStack
}
if g == nil || g.holds == nil {
return settings.RestoreHold{}, errors.New("no hold store is wired on this box")
}
resumeMu.Lock()
defer resumeMu.Unlock()
h, ok := g.holds.GetRestoreHold(stack)
if !ok {
return settings.RestoreHold{}, ErrNotHeld
}
cleared, err := g.holds.ClearRestoreHold(stack)
if err != nil {
return h, err
}
if !cleared {
return settings.RestoreHold{}, ErrNotHeld
}
if h.Reason == settings.HoldReasonPutBackPending {
g.dropFromMarker(stack)
g.logger.Printf("[WARN] [appstop] %s: its waiting put-back is DROPPED with the hold — the copy stays on disk at %s and is never poured back by itself", stack, h.SafetyDump)
}
return h, nil
}
@@ -425,9 +425,12 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
return
}
// R-1340: this branch is only ever an off-site restore's database replay (offbox_reconstitute's rollback).
h := settings.RestoreHold{
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
RestoreKind: settings.RestoreKindOffsite,
DBReplay: true,
}
if replayErr != nil {
h.ReplayError = replayErr.Error()
@@ -447,6 +450,15 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
}
}
// mixedHoldWhy (R-1340) is what a restore_mixed hold records about its cause (settings.RestoreHold's R-1340 fields).
type mixedHoldWhy struct {
kind string // settings.RestoreKind*
cause string // settings.MixedCause*
step string // the step that failed, with its error
dbReplay bool // the restore's own failure was its database replay
copyDir string // where the copy of the live volumes is kept; "" = none kept
}
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
type liveDefinition struct {
@@ -511,7 +523,11 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
// written back (when the snapshot's was written), the database service is stopped again, and the app is
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
// notification tells this case from R-379's double failure.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
//
// why (R-1340) is what the hold records about its cause, so the operator's mail tells the real story — this hold is
// written by three different branches now (a definition write, a put-back that failed at one of its steps), not
// only by option C's database replay.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition, why mixedHoldWhy) {
// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
@@ -519,7 +535,8 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
if m.settings == nil {
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
} else {
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
RestoreKind: why.kind, MixedCause: why.cause, FailedStep: why.step, DBReplay: why.dbReplay, SafetyDump: why.copyDir}
if replayErr != nil {
h.ReplayError = replayErr.Error()
}
@@ -557,7 +574,8 @@ func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, ca
if live != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
m.holdAppAfterMixedRestore(stack, cause, nil)
m.holdAppAfterMixedRestore(stack, cause, nil, mixedHoldWhy{kind: settings.RestoreKindOffsite, cause: settings.MixedCauseDefinitionWrite,
step: "writing the live app definition back failed: " + err.Error()})
return
}
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
@@ -1060,7 +1078,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
if definitionMoved {
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1095,7 +1113,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
if definitionMoved || nVols > 0 || undo != nil {
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
if sErr := restartStack(); sErr != nil {
@@ -1114,7 +1132,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Best-effort bring-up: a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1166,7 +1184,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed", settings.RestoreKindOffsite)
}
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
@@ -11,6 +11,7 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
//
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
// the household's sentence), so the operator's mail names the real cause.
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
filesPlaced int, restart func() error, holdKey string) error {
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
why := func(step string) mixedHoldWhy {
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
if c != nil && c.keep {
w.copyDir = c.dir
}
return w
}
if volumesMoved && c == nil {
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
return util.MsgError(holdKey, stack)
}
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.StopStack(stack); err != nil {
c.keepIfAny()
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
return util.MsgError(holdKey, stack)
}
if volumesMoved {
if err := m.putBack(stack, c); err != nil {
c.keep = true
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}
@@ -0,0 +1,133 @@
package backup
import (
"context"
"errors"
"io"
"log"
"os"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed,
// whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail
// (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on
// Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its
// DATABASE restore failed and was rolled back.
//
// RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from
// holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ /
// „kind """.
func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold {
t.Helper()
if m.settings == nil {
t.Fatal("fixture: no settings wired")
}
h, ok := m.settings.GetRestoreHold(stack)
if !ok {
t.Fatalf("%s is not held", stack)
}
return h
}
// The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails.
func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) {
m, vp, rig, _, notified, _ := r893PutBackFixture(t)
rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use")
p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
m.SetStackProvider(p)
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed restore must be surfaced")
}
h := holdOf(t, m, "immich")
if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack {
t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause)
}
if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") {
t.Errorf("failed step %q — want the stop and its error", h.FailedStep)
}
if h.DBReplay {
t.Error("a volume failure must not be recorded as a database replay")
}
if h.RestoreKind != settings.RestoreKindOffsite {
t.Errorf("restore kind %q, want offsite", h.RestoreKind)
}
if h.SafetyDump == "" {
t.Error("the kept copy's folder must be recorded")
} else if _, err := os.Stat(h.SafetyDump); err != nil {
t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err)
}
if *notified != 1 {
t.Errorf("hold notices %d, want 1", *notified)
}
}
// The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded.
func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) {
m, vp, _, _, _, _ := r893PutBackFixture(t)
m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2})
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed restore must be surfaced")
}
h := holdOf(t, m, "immich")
if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay {
t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay)
}
}
// R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved.
func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) {
m, _, _ := r1256DefFailFixture(t, 2)
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed definition write must be surfaced")
}
h := holdOf(t, m, "immich")
if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay {
t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay)
}
if !strings.Contains(h.FailedStep, "writing the live app definition back failed") {
t.Errorf("failed step %q", h.FailedStep)
}
}
// A LOCAL restore (from the app's own backup) records itself as local, never off-site.
func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) {
m, _, rig, _ := r1177Fixture(t)
if m.settings == nil {
sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
m.settings = sett
}
rig.unitErr = errors.New("tar: short read")
rig.putBackErr = errors.New("tar xf failed")
if _, err := m.RestoreFromRecoveryUnit("app"); err == nil {
t.Fatal("a failed put-back must be surfaced")
}
h := holdOf(t, m, "app")
if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay {
t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay)
}
if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") {
t.Errorf("failed step %q", h.FailedStep)
}
}
// A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause.
func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) {
r := newR1251Rig(t)
r.putErr = errors.New("compose down: timeout")
r.guard().Recover()
h, ok := r.sett.GetRestoreHold("immich")
if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack {
t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok)
}
if !strings.Contains(h.FailedStep, "compose down: timeout") {
t.Errorf("failed step %q", h.FailedStep)
}
}
+2 -1
View File
@@ -3,6 +3,7 @@ package backup
import (
"errors"
"fmt"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"os"
"path/filepath"
@@ -530,7 +531,7 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
// without one (the unit holds no volume) only a database replay that began can have changed data — and with no
// copy to pour back that is the R-379 half state: held, never started (operator ruling 2026-08-22).
putBack := func(cause error, holdKey string, moved bool) (UnitRestoreResult, error) {
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey)
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey, settings.RestoreKindLocal)
}
// R-353: the count is captured even when the replay errors — a partial replay is a fact the
// customer's sentence has to be built from, and discarding it on the error path is how Scenario C