clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s

- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING
  controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back
  from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type.
- --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid
  checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart).
- R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause,
  failed put-back step, database replay or not, kept copy) — never the option-C database story by default.
- R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
admin committed 2026-10-29 12:49:46 +01:00
1 parent c7bb776e00
commit 5b3b541602
25 files changed
+1809 -83

No files matched your search

@@ -119,7 +119,8 @@ func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
return false
}
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir,
MixedCause: settings.MixedCauseInterruptedPutBack, FailedStep: "putting the interrupted restore's copy back failed: " + cause.Error()}
if err := g.holds.SetRestoreHold(h); err != nil {
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
return false
+66
View File
@@ -0,0 +1,66 @@
package backup
import (
"errors"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// ── D64 = B (operator ruling 2026-10-29 12:10) + R-1343: the operator clears a hold IN THE RUNNING CONTROLLER ──────
//
// One door for both of the operator's routes — the hub's „Clear hold" (operator action clear_hold) and the box command
// `--clear-restore-hold`, which now asks the running controller over its local socket (cmd/controller/holdctl.go).
//
// WHY IN THIS PROCESS (R-1343, measured 2026-10-29 on Tester 1): the CLI used to run as a SECOND process that edited
// settings.json while the running controller kept the hold in its own in-memory Settings — and that controller's next
// save (after a hub report) wrote the hold back. Clearing through the running Settings removes the hold from memory
// AND saves the file at once, so no later save of any component can bring it back.
// Pinned by TestD64_ClearHoldStaysGoneAfterALaterSave (cmd/controller).
//
// EVERY REASON (D64 = B): the hold's reason does not matter — R-379 now reads „a person — on the box, or the operator
// on the hub". One reason needs a second act: a put-back WAITING for its drive (HoldReasonPutBackPending) is also
// recorded in the app-stop marker, and a controller start with the drive bound would pour that old copy back over
// whatever the app wrote after the clear. So the clear also drops the app from the marker; the copy stays on disk
// (nothing is deleted). It runs under resumeMu, so it can never interleave with a put-back of the same app.
// Pinned by TestD64_ClearPutBackPendingDropsTheMarkerEntry.
// ErrHoldBadStack: the argument is not a valid stack name. ErrNotHeld: the app has no hold. Both are refusals —
// nothing was changed.
var (
ErrHoldBadStack = errors.New("not a valid app name")
ErrNotHeld = errors.New("this app has no hold")
)
// IsValidHoldStack reports whether s may name an app for the hold commands (the backup package's strict
// isSafeStackName: letters, digits, '-' and '_', at most 64).
func IsValidHoldStack(s string) bool { return isSafeStackName(s) }
// ClearHoldByOperator lifts stack's hold, whatever its reason, in the hold store the running controller uses, and
// returns the hold it lifted. A waiting put-back is dropped from the app-stop marker too (see above). Refusals
// (ErrHoldBadStack, ErrNotHeld) change nothing.
func (g *AppStopGuard) ClearHoldByOperator(stack string) (settings.RestoreHold, error) {
if !isSafeStackName(stack) {
return settings.RestoreHold{}, ErrHoldBadStack
}
if g == nil || g.holds == nil {
return settings.RestoreHold{}, errors.New("no hold store is wired on this box")
}
resumeMu.Lock()
defer resumeMu.Unlock()
h, ok := g.holds.GetRestoreHold(stack)
if !ok {
return settings.RestoreHold{}, ErrNotHeld
}
cleared, err := g.holds.ClearRestoreHold(stack)
if err != nil {
return h, err
}
if !cleared {
return settings.RestoreHold{}, ErrNotHeld
}
if h.Reason == settings.HoldReasonPutBackPending {
g.dropFromMarker(stack)
g.logger.Printf("[WARN] [appstop] %s: its waiting put-back is DROPPED with the hold — the copy stays on disk at %s and is never poured back by itself", stack, h.SafetyDump)
}
return h, nil
}
@@ -425,9 +425,12 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
return
}
// R-1340: this branch is only ever an off-site restore's database replay (offbox_reconstitute's rollback).
h := settings.RestoreHold{
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
Stack: stack,
At: time.Now().UTC().Format(time.RFC3339),
RestoreKind: settings.RestoreKindOffsite,
DBReplay: true,
}
if replayErr != nil {
h.ReplayError = replayErr.Error()
@@ -447,6 +450,15 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
}
}
// mixedHoldWhy (R-1340) is what a restore_mixed hold records about its cause (settings.RestoreHold's R-1340 fields).
type mixedHoldWhy struct {
kind string // settings.RestoreKind*
cause string // settings.MixedCause*
step string // the step that failed, with its error
dbReplay bool // the restore's own failure was its database replay
copyDir string // where the copy of the live volumes is kept; "" = none kept
}
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
type liveDefinition struct {
@@ -511,7 +523,11 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
// written back (when the snapshot's was written), the database service is stopped again, and the app is
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
// notification tells this case from R-379's double failure.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
//
// why (R-1340) is what the hold records about its cause, so the operator's mail tells the real story — this hold is
// written by three different branches now (a definition write, a put-back that failed at one of its steps), not
// only by option C's database replay.
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition, why mixedHoldWhy) {
// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
@@ -519,7 +535,8 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
if m.settings == nil {
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
} else {
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
RestoreKind: why.kind, MixedCause: why.cause, FailedStep: why.step, DBReplay: why.dbReplay, SafetyDump: why.copyDir}
if replayErr != nil {
h.ReplayError = replayErr.Error()
}
@@ -557,7 +574,8 @@ func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, ca
if live != nil {
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
m.holdAppAfterMixedRestore(stack, cause, nil)
m.holdAppAfterMixedRestore(stack, cause, nil, mixedHoldWhy{kind: settings.RestoreKindOffsite, cause: settings.MixedCauseDefinitionWrite,
step: "writing the live app definition back failed: " + err.Error()})
return
}
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
@@ -1060,7 +1078,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
if definitionMoved {
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1095,7 +1113,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
if definitionMoved || nVols > 0 || undo != nil {
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
if sErr := restartStack(); sErr != nil {
@@ -1114,7 +1132,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
}
// Best-effort bring-up: a failed restore must not also be an outage.
if sErr := restartStack(); sErr != nil {
@@ -1166,7 +1184,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
if definitionMoved || res.VolumesReplayed > 0 {
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed", settings.RestoreKindOffsite)
}
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
@@ -11,6 +11,7 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
//
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
// the household's sentence), so the operator's mail names the real cause.
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
filesPlaced int, restart func() error, holdKey string) error {
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
why := func(step string) mixedHoldWhy {
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
if c != nil && c.keep {
w.copyDir = c.dir
}
return w
}
if volumesMoved && c == nil {
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
return util.MsgError(holdKey, stack)
}
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.StopStack(stack); err != nil {
c.keepIfAny()
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
return util.MsgError(holdKey, stack)
}
if volumesMoved {
if err := m.putBack(stack, c); err != nil {
c.keep = true
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
m.holdAppAfterMixedRestore(stack, cause, live)
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
return util.MsgError(holdKey, stack)
}
}
@@ -0,0 +1,133 @@
package backup
import (
"context"
"errors"
"io"
"log"
"os"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed,
// whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail
// (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on
// Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its
// DATABASE restore failed and was rolled back.
//
// RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from
// holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ /
// „kind """.
func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold {
t.Helper()
if m.settings == nil {
t.Fatal("fixture: no settings wired")
}
h, ok := m.settings.GetRestoreHold(stack)
if !ok {
t.Fatalf("%s is not held", stack)
}
return h
}
// The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails.
func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) {
m, vp, rig, _, notified, _ := r893PutBackFixture(t)
rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use")
p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
m.SetStackProvider(p)
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed restore must be surfaced")
}
h := holdOf(t, m, "immich")
if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack {
t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause)
}
if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") {
t.Errorf("failed step %q — want the stop and its error", h.FailedStep)
}
if h.DBReplay {
t.Error("a volume failure must not be recorded as a database replay")
}
if h.RestoreKind != settings.RestoreKindOffsite {
t.Errorf("restore kind %q, want offsite", h.RestoreKind)
}
if h.SafetyDump == "" {
t.Error("the kept copy's folder must be recorded")
} else if _, err := os.Stat(h.SafetyDump); err != nil {
t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err)
}
if *notified != 1 {
t.Errorf("hold notices %d, want 1", *notified)
}
}
// The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded.
func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) {
m, vp, _, _, _, _ := r893PutBackFixture(t)
m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2})
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed restore must be surfaced")
}
h := holdOf(t, m, "immich")
if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay {
t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay)
}
}
// R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved.
func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) {
m, _, _ := r1256DefFailFixture(t, 2)
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
t.Fatal("a failed definition write must be surfaced")
}
h := holdOf(t, m, "immich")
if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay {
t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay)
}
if !strings.Contains(h.FailedStep, "writing the live app definition back failed") {
t.Errorf("failed step %q", h.FailedStep)
}
}
// A LOCAL restore (from the app's own backup) records itself as local, never off-site.
func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) {
m, _, rig, _ := r1177Fixture(t)
if m.settings == nil {
sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
m.settings = sett
}
rig.unitErr = errors.New("tar: short read")
rig.putBackErr = errors.New("tar xf failed")
if _, err := m.RestoreFromRecoveryUnit("app"); err == nil {
t.Fatal("a failed put-back must be surfaced")
}
h := holdOf(t, m, "app")
if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay {
t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay)
}
if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") {
t.Errorf("failed step %q", h.FailedStep)
}
}
// A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause.
func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) {
r := newR1251Rig(t)
r.putErr = errors.New("compose down: timeout")
r.guard().Recover()
h, ok := r.sett.GetRestoreHold("immich")
if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack {
t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok)
}
if !strings.Contains(h.FailedStep, "compose down: timeout") {
t.Errorf("failed step %q", h.FailedStep)
}
}
+2 -1
View File
@@ -3,6 +3,7 @@ package backup
import (
"errors"
"fmt"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"os"
"path/filepath"
@@ -530,7 +531,7 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
// without one (the unit holds no volume) only a database replay that began can have changed data — and with no
// copy to pour back that is the R-379 half state: held, never started (operator ruling 2026-08-22).
putBack := func(cause error, holdKey string, moved bool) (UnitRestoreResult, error) {
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey)
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey, settings.RestoreKindLocal)
}
// R-353: the count is captured even when the replay errors — a partial replay is a fact the
// customer's sentence has to be built from, and discarding it on the error path is how Scenario C
+42 -4
View File
@@ -21,9 +21,11 @@ import (
// problem, main.go). The result rides the next report as `operator_action_results: [{id, outcome,
// message}]`; the hub lists an action until its result has arrived, then stops.
//
// THE LIST IS CLOSED. Four actions, and run_job names one of four jobs. Nothing here deletes data,
// THE LIST IS CLOSED. Five actions, and run_job names one of four jobs. Nothing here deletes data,
// starts a countdown or shortens one — `03` §4 asks for a signing key only to destroy or overwrite the
// only copy, and none of these does. An unknown action, an unknown job or an argument out of range is
// only copy, and none of these does. clear_hold (D64 = B, operator ruling 2026-10-29 12:10) lifts an app's
// hold of ANY reason with no operator signature: it deletes nothing and starts nothing — the app stays
// stopped until the household (or the operator) starts it, and a waiting put-back's copy stays on disk. An unknown action, an unknown job or an argument out of range is
// answered `refused` and NOTHING is called. Pinned by TestOpActions_ClosedList (which fails when an
// entry is added, so a new action is a decision, not an edit) and TestOpActions_UnknownIsRefused.
//
@@ -38,6 +40,7 @@ const (
OpAbandonStop = "abandon_stop"
OpAbandonExtend = "abandon_extend"
OpRunJob = "run_job"
OpClearHold = "clear_hold"
OutcomeDone = "done"
OutcomeRefused = "refused"
@@ -57,7 +60,7 @@ const (
)
// operatorActionNames is THE closed list. Adding an entry fails TestOpActions_ClosedList by design.
var operatorActionNames = []string{OpOffsiteBackupNow, OpAbandonStop, OpAbandonExtend, OpRunJob}
var operatorActionNames = []string{OpOffsiteBackupNow, OpAbandonStop, OpAbandonExtend, OpRunJob, OpClearHold}
// operatorJobNames is the fixed set run_job may name — the scheduler's own job names (main.go).
var operatorJobNames = []string{"fill-watch", "offsite-integrity", "offsite-proof", "disk-health-check"}
@@ -83,6 +86,21 @@ func contains(list []string, v string) bool {
return false
}
// validHoldStack is clear_hold's argument check: the same alphabet as backup.isSafeStackName (report cannot import
// backup; REUSE.md „Safe-name validators").
func validHoldStack(s string) bool {
if s == "" || len(s) > 64 {
return false
}
for _, c := range s {
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
continue
}
return false
}
return true
}
// ErrActionRefused, wrapped by a handler, turns its error into a `refused` outcome (the action could
// not START — e.g. an off-site run already in flight) instead of `failed` (it started and went wrong).
var ErrActionRefused = errors.New("refused")
@@ -101,7 +119,7 @@ type OperatorActionResult struct {
Message string `json:"message"`
}
// OperatorActionHandlers are the four doors, wired by main.go. A nil handler answers `refused`
// OperatorActionHandlers are the five doors, wired by main.go. A nil handler answers `refused`
// („not available on this box").
type OperatorActionHandlers struct {
// OffsiteBackupNow returns a refusal sentence when the run may not start (the household's
@@ -111,6 +129,9 @@ type OperatorActionHandlers struct {
AbandonExtend func(days int) (time.Time, error)
// RunJob is Scheduler.RunNow: an error = not started (unknown/running); the channel = the job's end.
RunJob func(name string) (<-chan error, error)
// ClearHold lifts the app's hold in the RUNNING controller (D64 = B, R-1343) and returns the result
// sentence. actionID names the action in the box's log. An error wrapping ErrActionRefused = nothing changed.
ClearHold func(stack string, actionID int64) (string, error)
// ResultReady sends an out-of-cycle report so a result reaches the hub in seconds (optional).
ResultReady func()
}
@@ -317,6 +338,23 @@ func (o *OperatorActions) dispatch(a OperatorAction) {
}
o.finish(a, outcomeOf(jerr), msg)
}()
case OpClearHold:
// The argument is checked HERE, before any handler is touched (the closed list's rule): one app name,
// the backup package's strict alphabet (letters, digits, '-' and '_', at most 64).
if !validHoldStack(a.Arg) {
o.finish(a, OutcomeRefused, "the argument must be one app name (letters, digits, '-' and '_')")
return
}
if o.h.ClearHold == nil {
o.finish(a, OutcomeRefused, "not available on this box")
return
}
msg, err := o.h.ClearHold(a.Arg, a.ID)
if err != nil {
o.finish(a, outcomeOf(err), err.Error())
return
}
o.finish(a, OutcomeDone, msg)
default:
o.finish(a, OutcomeRefused, "unknown action")
}
+79 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"sync"
@@ -15,9 +16,11 @@ import (
// The list is CLOSED. This test fails when an action or a job is added, by design: a new entry is an
// operator decision (no action may delete data, start a countdown or shorten one), not an edit. The
// hub pins the same four in its own test (hub internal/store opactions_test.go).
// hub pins the same list in its own test (hub internal/store opactions_test.go).
// 2026-10-29: clear_hold added deliberately — operator ruling D64 = B (12:10): the hub's „Clear hold" works for
// every hold reason with no operator signature. It deletes nothing and starts nothing.
func TestOpActions_ClosedList(t *testing.T) {
if got, want := OperatorActionNames(), []string{"abandon_extend", "abandon_stop", "offsite_backup_now", "run_job"}; !reflect.DeepEqual(got, want) {
if got, want := OperatorActionNames(), []string{"abandon_extend", "abandon_stop", "clear_hold", "offsite_backup_now", "run_job"}; !reflect.DeepEqual(got, want) {
t.Fatalf("operator actions = %v, want exactly %v — a new action needs the operator's word (decision 185)", got, want)
}
if got, want := OperatorJobNames(), []string{"disk-health-check", "fill-watch", "offsite-integrity", "offsite-proof"}; !reflect.DeepEqual(got, want) {
@@ -56,6 +59,10 @@ func recordingHandlers(c *calls) OperatorActionHandlers {
close(ch)
return ch, nil
},
ClearHold: func(stack string, id int64) (string, error) {
c.add("clear:" + stack)
return "the hold on " + stack + " is cleared", nil
},
}
}
@@ -197,3 +204,73 @@ func TestOpActions_WireShapes(t *testing.T) {
t.Fatalf("report carried %s", b)
}
}
// clear_hold (D64 = B): the argument is one app name in the strict alphabet — never a path, never empty, never two
// names. A bad one is refused and the handler is NOT called (the closed list's rule: validate before any door).
// RED-PROOF (2026-10-29): with validHoldStack returning true, this fails on „a refused action called [clear:…]".
func TestOpActions_ClearHoldBadArgumentIsRefusedAndCallsNothing(t *testing.T) {
c := &calls{}
o := NewOperatorActions(context.Background(), recordingHandlers(c), nil)
list := []OperatorAction{
{ID: 1, Action: OpClearHold, Arg: ""},
{ID: 2, Action: OpClearHold, Arg: "../settings"},
{ID: 3, Action: OpClearHold, Arg: "immich paperless"},
{ID: 4, Action: OpClearHold, Arg: "immich/x"},
{ID: 5, Action: OpClearHold, Arg: strings.Repeat("a", 65)},
{ID: 6, Action: OpClearHold, Arg: "immich\n"},
{ID: 7, Action: OpClearHold, Arg: "."},
}
o.Reconcile(list)
if got := c.get(); len(got) != 0 {
t.Fatalf("a refused action called %v", got)
}
for _, a := range list {
if r := resultFor(t, o, a.ID); r.Outcome != OutcomeRefused || !strings.Contains(r.Message, "one app name") {
t.Errorf("#%d clear_hold(%q): %+v, want refused naming the rule", a.ID, a.Arg, r)
}
}
// control: a good name does reach the handler
o.Reconcile([]OperatorAction{{ID: 8, Action: OpClearHold, Arg: "paperless-ngx_2"}})
if got := c.get(); len(got) != 1 || got[0] != "clear:paperless-ngx_2" {
t.Fatalf("a valid name must reach the handler, calls %v", got)
}
}
// clear_hold (D64 = B): a valid name reaches the handler with the action's id, and its sentence is the result; a
// handler refusal (the app has no hold) is answered refused, an error that is not a refusal failed.
// RED-PROOF (2026-10-29): with the OpClearHold case removed from dispatch, this fails on „unknown action".
func TestOpActions_ClearHoldDoneRefusedFailed(t *testing.T) {
var gotStack string
var gotID int64
h := OperatorActionHandlers{ClearHold: func(stack string, id int64) (string, error) {
gotStack, gotID = stack, id
switch stack {
case "notheld":
return "", fmt.Errorf("%w: notheld has no hold — nothing was changed", ErrActionRefused)
case "broken":
return "", errors.New("saving settings: disk full")
}
return "the hold on " + stack + " is cleared (it was: held after a failed update)", nil
}}
o := NewOperatorActions(context.Background(), h, nil)
o.Reconcile([]OperatorAction{{ID: 7, Action: OpClearHold, Arg: "kimai"}})
if gotStack != "kimai" || gotID != 7 {
t.Fatalf("handler called with (%q, %d), want (kimai, 7)", gotStack, gotID)
}
if r := resultFor(t, o, 7); r.Outcome != OutcomeDone || r.Message != "the hold on kimai is cleared (it was: held after a failed update)" {
t.Fatalf("result %+v", r)
}
o.Reconcile([]OperatorAction{{ID: 8, Action: OpClearHold, Arg: "notheld"}, {ID: 9, Action: OpClearHold, Arg: "broken"}})
if r := resultFor(t, o, 8); r.Outcome != OutcomeRefused || !strings.Contains(r.Message, "has no hold") {
t.Errorf("not held: %+v", r)
}
if r := resultFor(t, o, 9); r.Outcome != OutcomeFailed {
t.Errorf("a save error must be failed, not refused: %+v", r)
}
// nil handler → refused, not a crash
o2 := NewOperatorActions(context.Background(), OperatorActionHandlers{}, nil)
o2.Reconcile([]OperatorAction{{ID: 1, Action: OpClearHold, Arg: "kimai"}})
if r := resultFor(t, o2, 1); r.Outcome != OutcomeRefused {
t.Errorf("nil handler: %+v", r)
}
}
+28
View File
@@ -1982,8 +1982,36 @@ type RestoreHold struct {
// first stop or a repeat within 24 h (Trip >= 2 → the sentence says support is informed).
UnhealthyKind string `json:"unhealthy_kind,omitempty"`
Trip int `json:"trip,omitempty"`
// R-1340 (2026-10-29): what put a restore hold in place, so the operator's „HELD STOPPED" mail tells the real
// story instead of the option-C database story for every cause. All empty on a hold written before them: the mail
// then names only what the record carries (cmd/controller heldStoppedOperatorText).
//
// RestoreKind is which restore: RestoreKindOffsite or RestoreKindLocal.
RestoreKind string `json:"restore_kind,omitempty"`
// MixedCause (restore_mixed only) is the branch that held the app: MixedCause*.
MixedCause string `json:"mixed_cause,omitempty"`
// FailedStep is the put-back step that failed, with its error (MixedCausePutBack).
FailedStep string `json:"failed_step,omitempty"`
// DBReplay: the restore's own failure was its database replay. False = the app's database was never touched
// by the failure, so the mail never says „database".
DBReplay bool `json:"db_replay,omitempty"`
}
// Restore kinds and mixed-hold causes (R-1340). See RestoreHold.RestoreKind / MixedCause.
const (
RestoreKindOffsite = "offsite"
RestoreKindLocal = "local"
// MixedCauseDefinitionWrite: the backup's definition could not be written (no data moved yet) and writing the
// live definition back failed too (R-1256 H4).
MixedCauseDefinitionWrite = "definition_write"
// MixedCausePutBack: the restore failed and putting the live data back failed at FailedStep (D20/R-1256 H5).
MixedCausePutBack = "put_back"
// MixedCauseInterruptedPutBack: a restore cut by a stop could not be put back at the next start (R-1251 H2).
MixedCauseInterruptedPutBack = "interrupted_put_back"
)
// Hold reasons. See RestoreHold.Reason.
const (
HoldReasonRestoreFailed = "" // R-379/R-380: a restore AND its rollback failed
@@ -0,0 +1,46 @@
package stacks
import (
"io"
"log"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)
// D64 = B / R-1343: when the operator clears a hold, the ONE held flag kept outside the hold store — „the last update
// ended held" — goes too. Left set, the crash-loop observer and the nightly update leg kept skipping an app whose hold
// was gone, until the controller restarted.
// RED-PROOF (2026-10-29): with ForgetUpdateHeld's body reduced to `return false`, this fails on „a held update must be forgotten".
func TestD64_ForgetUpdateHeld(t *testing.T) {
m := &Manager{cfg: &config.Config{}, logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
"kimai": {Name: "kimai", Deployed: true},
"running": {Name: "running", Deployed: true, Updating: true},
}}
m.finishUpdateKey("kimai", UpdatePhaseFailed, UpdateErrorKeyHeld, "a kimai leállítva marad")
m.markUpdateHeld("kimai")
m.markUpdateHeld("running")
if !m.ForgetUpdateHeld("kimai") {
t.Fatal("a held update must be forgotten")
}
st, _ := m.GetStack("kimai")
if st.updateHeld || st.UpdateErrorKey != "" || st.UpdateError != "" || st.UpdatePhase != "" {
t.Fatalf("held state left: held=%v key=%q err=%q phase=%q", st.updateHeld, st.UpdateErrorKey, st.UpdateError, st.UpdatePhase)
}
// The two readers of the flag (ObserveUnhealthy's skip, unhealthy.go; legCandidate's LegSkipHeld, unattended.go)
// read the stored stack — not called here, because ObserveUnhealthy reaches docker (R-650).
m.mu.RLock()
skipped := m.stacks["kimai"].updateHeld || m.stacks["kimai"].UpdateErrorKey == UpdateErrorKeyHeld
m.mu.RUnlock()
if skipped {
t.Fatal("the observer and the update leg still skip kimai")
}
// A running update is never touched; an app with nothing held reports false.
if m.ForgetUpdateHeld("running") {
t.Fatal("a running update must not be touched")
}
if m.ForgetUpdateHeld("kimai") || m.ForgetUpdateHeld("nosuch") {
t.Fatal("nothing held → false")
}
}
+19
View File
@@ -209,6 +209,25 @@ func (m *Manager) ClearUpdateState(name string) {
s.HealthProbe = nil
}
// ForgetUpdateHeld (D64 = B, R-1343) forgets that the app's last update ended HELD, once the hold itself is cleared
// by the operator. updateHeld is the ONE held flag kept outside the hold store: it makes the crash-loop observer
// (ObserveUnhealthy) and the nightly update leg (legCandidate) skip the app, and keeps the hold's sentence as the
// update error. Left set, an app whose hold was cleared would stay unobserved and never be updated by night until
// the controller restarts. A running update is not touched. Returns whether anything was forgotten.
// Pinned by TestD64_ForgetUpdateHeld (this package) and TestD64_ClearHoldForgetsTheUpdateHeldFlag (cmd/controller).
func (m *Manager) ForgetUpdateHeld(name string) bool {
m.mu.Lock()
defer m.mu.Unlock()
s, ok := m.stacks[name]
if !ok || s.Updating || (!s.updateHeld && s.UpdateErrorKey != UpdateErrorKeyHeld) {
return false
}
s.updateHeld = false
s.UpdatePhase, s.UpdatePhaseLabel = "", ""
s.UpdateError, s.UpdateErrorKey, s.UpdateErrorArgs = "", "", nil
return true
}
func (m *Manager) markUpdateHeld(name string) {
m.mu.Lock()
if s, ok := m.stacks[name]; ok {