clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type. - --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart). - R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause, failed put-back step, database replay or not, kept copy) — never the option-C database story by default. - R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
1 parent
c7bb776e00
commit
5b3b541602
25 files changed
+1809
-83
No files matched your search
@@ -119,7 +119,8 @@ func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
|
||||
return false
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
|
||||
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
|
||||
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir,
|
||||
MixedCause: settings.MixedCauseInterruptedPutBack, FailedStep: "putting the interrupted restore's copy back failed: " + cause.Error()}
|
||||
if err := g.holds.SetRestoreHold(h); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── D64 = B (operator ruling 2026-10-29 12:10) + R-1343: the operator clears a hold IN THE RUNNING CONTROLLER ──────
|
||||
//
|
||||
// One door for both of the operator's routes — the hub's „Clear hold" (operator action clear_hold) and the box command
|
||||
// `--clear-restore-hold`, which now asks the running controller over its local socket (cmd/controller/holdctl.go).
|
||||
//
|
||||
// WHY IN THIS PROCESS (R-1343, measured 2026-10-29 on Tester 1): the CLI used to run as a SECOND process that edited
|
||||
// settings.json while the running controller kept the hold in its own in-memory Settings — and that controller's next
|
||||
// save (after a hub report) wrote the hold back. Clearing through the running Settings removes the hold from memory
|
||||
// AND saves the file at once, so no later save of any component can bring it back.
|
||||
// Pinned by TestD64_ClearHoldStaysGoneAfterALaterSave (cmd/controller).
|
||||
//
|
||||
// EVERY REASON (D64 = B): the hold's reason does not matter — R-379 now reads „a person — on the box, or the operator
|
||||
// on the hub". One reason needs a second act: a put-back WAITING for its drive (HoldReasonPutBackPending) is also
|
||||
// recorded in the app-stop marker, and a controller start with the drive bound would pour that old copy back over
|
||||
// whatever the app wrote after the clear. So the clear also drops the app from the marker; the copy stays on disk
|
||||
// (nothing is deleted). It runs under resumeMu, so it can never interleave with a put-back of the same app.
|
||||
// Pinned by TestD64_ClearPutBackPendingDropsTheMarkerEntry.
|
||||
|
||||
// ErrHoldBadStack: the argument is not a valid stack name. ErrNotHeld: the app has no hold. Both are refusals —
|
||||
// nothing was changed.
|
||||
var (
|
||||
ErrHoldBadStack = errors.New("not a valid app name")
|
||||
ErrNotHeld = errors.New("this app has no hold")
|
||||
)
|
||||
|
||||
// IsValidHoldStack reports whether s may name an app for the hold commands (the backup package's strict
|
||||
// isSafeStackName: letters, digits, '-' and '_', at most 64).
|
||||
func IsValidHoldStack(s string) bool { return isSafeStackName(s) }
|
||||
|
||||
// ClearHoldByOperator lifts stack's hold, whatever its reason, in the hold store the running controller uses, and
|
||||
// returns the hold it lifted. A waiting put-back is dropped from the app-stop marker too (see above). Refusals
|
||||
// (ErrHoldBadStack, ErrNotHeld) change nothing.
|
||||
func (g *AppStopGuard) ClearHoldByOperator(stack string) (settings.RestoreHold, error) {
|
||||
if !isSafeStackName(stack) {
|
||||
return settings.RestoreHold{}, ErrHoldBadStack
|
||||
}
|
||||
if g == nil || g.holds == nil {
|
||||
return settings.RestoreHold{}, errors.New("no hold store is wired on this box")
|
||||
}
|
||||
resumeMu.Lock()
|
||||
defer resumeMu.Unlock()
|
||||
h, ok := g.holds.GetRestoreHold(stack)
|
||||
if !ok {
|
||||
return settings.RestoreHold{}, ErrNotHeld
|
||||
}
|
||||
cleared, err := g.holds.ClearRestoreHold(stack)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
if !cleared {
|
||||
return settings.RestoreHold{}, ErrNotHeld
|
||||
}
|
||||
if h.Reason == settings.HoldReasonPutBackPending {
|
||||
g.dropFromMarker(stack)
|
||||
g.logger.Printf("[WARN] [appstop] %s: its waiting put-back is DROPPED with the hold — the copy stays on disk at %s and is never poured back by itself", stack, h.SafetyDump)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -425,9 +425,12 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
|
||||
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
|
||||
return
|
||||
}
|
||||
// R-1340: this branch is only ever an off-site restore's database replay (offbox_reconstitute's rollback).
|
||||
h := settings.RestoreHold{
|
||||
Stack: stack,
|
||||
At: time.Now().UTC().Format(time.RFC3339),
|
||||
Stack: stack,
|
||||
At: time.Now().UTC().Format(time.RFC3339),
|
||||
RestoreKind: settings.RestoreKindOffsite,
|
||||
DBReplay: true,
|
||||
}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
@@ -447,6 +450,15 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
|
||||
}
|
||||
}
|
||||
|
||||
// mixedHoldWhy (R-1340) is what a restore_mixed hold records about its cause (settings.RestoreHold's R-1340 fields).
|
||||
type mixedHoldWhy struct {
|
||||
kind string // settings.RestoreKind*
|
||||
cause string // settings.MixedCause*
|
||||
step string // the step that failed, with its error
|
||||
dbReplay bool // the restore's own failure was its database replay
|
||||
copyDir string // where the copy of the live volumes is kept; "" = none kept
|
||||
}
|
||||
|
||||
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
|
||||
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
|
||||
type liveDefinition struct {
|
||||
@@ -511,7 +523,11 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
|
||||
// written back (when the snapshot's was written), the database service is stopped again, and the app is
|
||||
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
|
||||
// notification tells this case from R-379's double failure.
|
||||
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
|
||||
//
|
||||
// why (R-1340) is what the hold records about its cause, so the operator's mail tells the real story — this hold is
|
||||
// written by three different branches now (a definition write, a put-back that failed at one of its steps), not
|
||||
// only by option C's database replay.
|
||||
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition, why mixedHoldWhy) {
|
||||
// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
|
||||
// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
|
||||
// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
|
||||
@@ -519,7 +535,8 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
|
||||
if m.settings == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
|
||||
} else {
|
||||
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
|
||||
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
|
||||
RestoreKind: why.kind, MixedCause: why.cause, FailedStep: why.step, DBReplay: why.dbReplay, SafetyDump: why.copyDir}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
}
|
||||
@@ -557,7 +574,8 @@ func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, ca
|
||||
if live != nil {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
|
||||
m.holdAppAfterMixedRestore(stack, cause, nil)
|
||||
m.holdAppAfterMixedRestore(stack, cause, nil, mixedHoldWhy{kind: settings.RestoreKindOffsite, cause: settings.MixedCauseDefinitionWrite,
|
||||
step: "writing the live app definition back failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
|
||||
@@ -1060,7 +1078,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
|
||||
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
|
||||
if definitionMoved {
|
||||
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1095,7 +1113,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
|
||||
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
|
||||
if definitionMoved || nVols > 0 || undo != nil {
|
||||
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1114,7 +1132,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
|
||||
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Best-effort bring-up: a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1166,7 +1184,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
|
||||
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
|
||||
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
|
||||
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
|
||||
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
|
||||
//
|
||||
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
|
||||
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
|
||||
// the household's sentence), so the operator's mail names the real cause.
|
||||
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
|
||||
filesPlaced int, restart func() error, holdKey string) error {
|
||||
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
|
||||
why := func(step string) mixedHoldWhy {
|
||||
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
|
||||
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
|
||||
if c != nil && c.keep {
|
||||
w.copyDir = c.dir
|
||||
}
|
||||
return w
|
||||
}
|
||||
if volumesMoved && c == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
|
||||
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.StopStack(stack); err != nil {
|
||||
c.keepIfAny()
|
||||
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
if volumesMoved {
|
||||
if err := m.putBack(stack, c); err != nil {
|
||||
c.keep = true
|
||||
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
|
||||
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed,
|
||||
// whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail
|
||||
// (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on
|
||||
// Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its
|
||||
// DATABASE restore failed and was rolled back.
|
||||
//
|
||||
// RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from
|
||||
// holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ /
|
||||
// „kind """.
|
||||
|
||||
func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold {
|
||||
t.Helper()
|
||||
if m.settings == nil {
|
||||
t.Fatal("fixture: no settings wired")
|
||||
}
|
||||
h, ok := m.settings.GetRestoreHold(stack)
|
||||
if !ok {
|
||||
t.Fatalf("%s is not held", stack)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails.
|
||||
func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) {
|
||||
m, vp, rig, _, notified, _ := r893PutBackFixture(t)
|
||||
rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use")
|
||||
p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
|
||||
m.SetStackProvider(p)
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed restore must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack {
|
||||
t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") {
|
||||
t.Errorf("failed step %q — want the stop and its error", h.FailedStep)
|
||||
}
|
||||
if h.DBReplay {
|
||||
t.Error("a volume failure must not be recorded as a database replay")
|
||||
}
|
||||
if h.RestoreKind != settings.RestoreKindOffsite {
|
||||
t.Errorf("restore kind %q, want offsite", h.RestoreKind)
|
||||
}
|
||||
if h.SafetyDump == "" {
|
||||
t.Error("the kept copy's folder must be recorded")
|
||||
} else if _, err := os.Stat(h.SafetyDump); err != nil {
|
||||
t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err)
|
||||
}
|
||||
if *notified != 1 {
|
||||
t.Errorf("hold notices %d, want 1", *notified)
|
||||
}
|
||||
}
|
||||
|
||||
// The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded.
|
||||
func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) {
|
||||
m, vp, _, _, _, _ := r893PutBackFixture(t)
|
||||
m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2})
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed restore must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay {
|
||||
t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved.
|
||||
func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) {
|
||||
m, _, _ := r1256DefFailFixture(t, 2)
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed definition write must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay {
|
||||
t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "writing the live app definition back failed") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
|
||||
// A LOCAL restore (from the app's own backup) records itself as local, never off-site.
|
||||
func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) {
|
||||
m, _, rig, _ := r1177Fixture(t)
|
||||
if m.settings == nil {
|
||||
sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.settings = sett
|
||||
}
|
||||
rig.unitErr = errors.New("tar: short read")
|
||||
rig.putBackErr = errors.New("tar xf failed")
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err == nil {
|
||||
t.Fatal("a failed put-back must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "app")
|
||||
if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay {
|
||||
t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
|
||||
// A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause.
|
||||
func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.putErr = errors.New("compose down: timeout")
|
||||
r.guard().Recover()
|
||||
h, ok := r.sett.GetRestoreHold("immich")
|
||||
if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack {
|
||||
t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "compose down: timeout") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package backup
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -530,7 +531,7 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
|
||||
// without one (the unit holds no volume) only a database replay that began can have changed data — and with no
|
||||
// copy to pour back that is the R-379 half state: held, never started (operator ruling 2026-08-22).
|
||||
putBack := func(cause error, holdKey string, moved bool) (UnitRestoreResult, error) {
|
||||
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey)
|
||||
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey, settings.RestoreKindLocal)
|
||||
}
|
||||
// R-353: the count is captured even when the replay errors — a partial replay is a fact the
|
||||
// customer's sentence has to be built from, and discarding it on the error path is how Scenario C
|
||||
|
||||
@@ -21,9 +21,11 @@ import (
|
||||
// problem, main.go). The result rides the next report as `operator_action_results: [{id, outcome,
|
||||
// message}]`; the hub lists an action until its result has arrived, then stops.
|
||||
//
|
||||
// THE LIST IS CLOSED. Four actions, and run_job names one of four jobs. Nothing here deletes data,
|
||||
// THE LIST IS CLOSED. Five actions, and run_job names one of four jobs. Nothing here deletes data,
|
||||
// starts a countdown or shortens one — `03` §4 asks for a signing key only to destroy or overwrite the
|
||||
// only copy, and none of these does. An unknown action, an unknown job or an argument out of range is
|
||||
// only copy, and none of these does. clear_hold (D64 = B, operator ruling 2026-10-29 12:10) lifts an app's
|
||||
// hold of ANY reason with no operator signature: it deletes nothing and starts nothing — the app stays
|
||||
// stopped until the household (or the operator) starts it, and a waiting put-back's copy stays on disk. An unknown action, an unknown job or an argument out of range is
|
||||
// answered `refused` and NOTHING is called. Pinned by TestOpActions_ClosedList (which fails when an
|
||||
// entry is added, so a new action is a decision, not an edit) and TestOpActions_UnknownIsRefused.
|
||||
//
|
||||
@@ -38,6 +40,7 @@ const (
|
||||
OpAbandonStop = "abandon_stop"
|
||||
OpAbandonExtend = "abandon_extend"
|
||||
OpRunJob = "run_job"
|
||||
OpClearHold = "clear_hold"
|
||||
|
||||
OutcomeDone = "done"
|
||||
OutcomeRefused = "refused"
|
||||
@@ -57,7 +60,7 @@ const (
|
||||
)
|
||||
|
||||
// operatorActionNames is THE closed list. Adding an entry fails TestOpActions_ClosedList by design.
|
||||
var operatorActionNames = []string{OpOffsiteBackupNow, OpAbandonStop, OpAbandonExtend, OpRunJob}
|
||||
var operatorActionNames = []string{OpOffsiteBackupNow, OpAbandonStop, OpAbandonExtend, OpRunJob, OpClearHold}
|
||||
|
||||
// operatorJobNames is the fixed set run_job may name — the scheduler's own job names (main.go).
|
||||
var operatorJobNames = []string{"fill-watch", "offsite-integrity", "offsite-proof", "disk-health-check"}
|
||||
@@ -83,6 +86,21 @@ func contains(list []string, v string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// validHoldStack is clear_hold's argument check: the same alphabet as backup.isSafeStackName (report cannot import
|
||||
// backup; REUSE.md „Safe-name validators").
|
||||
func validHoldStack(s string) bool {
|
||||
if s == "" || len(s) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ErrActionRefused, wrapped by a handler, turns its error into a `refused` outcome (the action could
|
||||
// not START — e.g. an off-site run already in flight) instead of `failed` (it started and went wrong).
|
||||
var ErrActionRefused = errors.New("refused")
|
||||
@@ -101,7 +119,7 @@ type OperatorActionResult struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// OperatorActionHandlers are the four doors, wired by main.go. A nil handler answers `refused`
|
||||
// OperatorActionHandlers are the five doors, wired by main.go. A nil handler answers `refused`
|
||||
// („not available on this box").
|
||||
type OperatorActionHandlers struct {
|
||||
// OffsiteBackupNow returns a refusal sentence when the run may not start (the household's
|
||||
@@ -111,6 +129,9 @@ type OperatorActionHandlers struct {
|
||||
AbandonExtend func(days int) (time.Time, error)
|
||||
// RunJob is Scheduler.RunNow: an error = not started (unknown/running); the channel = the job's end.
|
||||
RunJob func(name string) (<-chan error, error)
|
||||
// ClearHold lifts the app's hold in the RUNNING controller (D64 = B, R-1343) and returns the result
|
||||
// sentence. actionID names the action in the box's log. An error wrapping ErrActionRefused = nothing changed.
|
||||
ClearHold func(stack string, actionID int64) (string, error)
|
||||
// ResultReady sends an out-of-cycle report so a result reaches the hub in seconds (optional).
|
||||
ResultReady func()
|
||||
}
|
||||
@@ -317,6 +338,23 @@ func (o *OperatorActions) dispatch(a OperatorAction) {
|
||||
}
|
||||
o.finish(a, outcomeOf(jerr), msg)
|
||||
}()
|
||||
case OpClearHold:
|
||||
// The argument is checked HERE, before any handler is touched (the closed list's rule): one app name,
|
||||
// the backup package's strict alphabet (letters, digits, '-' and '_', at most 64).
|
||||
if !validHoldStack(a.Arg) {
|
||||
o.finish(a, OutcomeRefused, "the argument must be one app name (letters, digits, '-' and '_')")
|
||||
return
|
||||
}
|
||||
if o.h.ClearHold == nil {
|
||||
o.finish(a, OutcomeRefused, "not available on this box")
|
||||
return
|
||||
}
|
||||
msg, err := o.h.ClearHold(a.Arg, a.ID)
|
||||
if err != nil {
|
||||
o.finish(a, outcomeOf(err), err.Error())
|
||||
return
|
||||
}
|
||||
o.finish(a, OutcomeDone, msg)
|
||||
default:
|
||||
o.finish(a, OutcomeRefused, "unknown action")
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -15,9 +16,11 @@ import (
|
||||
|
||||
// The list is CLOSED. This test fails when an action or a job is added, by design: a new entry is an
|
||||
// operator decision (no action may delete data, start a countdown or shorten one), not an edit. The
|
||||
// hub pins the same four in its own test (hub internal/store opactions_test.go).
|
||||
// hub pins the same list in its own test (hub internal/store opactions_test.go).
|
||||
// 2026-10-29: clear_hold added deliberately — operator ruling D64 = B (12:10): the hub's „Clear hold" works for
|
||||
// every hold reason with no operator signature. It deletes nothing and starts nothing.
|
||||
func TestOpActions_ClosedList(t *testing.T) {
|
||||
if got, want := OperatorActionNames(), []string{"abandon_extend", "abandon_stop", "offsite_backup_now", "run_job"}; !reflect.DeepEqual(got, want) {
|
||||
if got, want := OperatorActionNames(), []string{"abandon_extend", "abandon_stop", "clear_hold", "offsite_backup_now", "run_job"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("operator actions = %v, want exactly %v — a new action needs the operator's word (decision 185)", got, want)
|
||||
}
|
||||
if got, want := OperatorJobNames(), []string{"disk-health-check", "fill-watch", "offsite-integrity", "offsite-proof"}; !reflect.DeepEqual(got, want) {
|
||||
@@ -56,6 +59,10 @@ func recordingHandlers(c *calls) OperatorActionHandlers {
|
||||
close(ch)
|
||||
return ch, nil
|
||||
},
|
||||
ClearHold: func(stack string, id int64) (string, error) {
|
||||
c.add("clear:" + stack)
|
||||
return "the hold on " + stack + " is cleared", nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,3 +204,73 @@ func TestOpActions_WireShapes(t *testing.T) {
|
||||
t.Fatalf("report carried %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
// clear_hold (D64 = B): the argument is one app name in the strict alphabet — never a path, never empty, never two
|
||||
// names. A bad one is refused and the handler is NOT called (the closed list's rule: validate before any door).
|
||||
// RED-PROOF (2026-10-29): with validHoldStack returning true, this fails on „a refused action called [clear:…]".
|
||||
func TestOpActions_ClearHoldBadArgumentIsRefusedAndCallsNothing(t *testing.T) {
|
||||
c := &calls{}
|
||||
o := NewOperatorActions(context.Background(), recordingHandlers(c), nil)
|
||||
list := []OperatorAction{
|
||||
{ID: 1, Action: OpClearHold, Arg: ""},
|
||||
{ID: 2, Action: OpClearHold, Arg: "../settings"},
|
||||
{ID: 3, Action: OpClearHold, Arg: "immich paperless"},
|
||||
{ID: 4, Action: OpClearHold, Arg: "immich/x"},
|
||||
{ID: 5, Action: OpClearHold, Arg: strings.Repeat("a", 65)},
|
||||
{ID: 6, Action: OpClearHold, Arg: "immich\n"},
|
||||
{ID: 7, Action: OpClearHold, Arg: "."},
|
||||
}
|
||||
o.Reconcile(list)
|
||||
if got := c.get(); len(got) != 0 {
|
||||
t.Fatalf("a refused action called %v", got)
|
||||
}
|
||||
for _, a := range list {
|
||||
if r := resultFor(t, o, a.ID); r.Outcome != OutcomeRefused || !strings.Contains(r.Message, "one app name") {
|
||||
t.Errorf("#%d clear_hold(%q): %+v, want refused naming the rule", a.ID, a.Arg, r)
|
||||
}
|
||||
}
|
||||
// control: a good name does reach the handler
|
||||
o.Reconcile([]OperatorAction{{ID: 8, Action: OpClearHold, Arg: "paperless-ngx_2"}})
|
||||
if got := c.get(); len(got) != 1 || got[0] != "clear:paperless-ngx_2" {
|
||||
t.Fatalf("a valid name must reach the handler, calls %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// clear_hold (D64 = B): a valid name reaches the handler with the action's id, and its sentence is the result; a
|
||||
// handler refusal (the app has no hold) is answered refused, an error that is not a refusal failed.
|
||||
// RED-PROOF (2026-10-29): with the OpClearHold case removed from dispatch, this fails on „unknown action".
|
||||
func TestOpActions_ClearHoldDoneRefusedFailed(t *testing.T) {
|
||||
var gotStack string
|
||||
var gotID int64
|
||||
h := OperatorActionHandlers{ClearHold: func(stack string, id int64) (string, error) {
|
||||
gotStack, gotID = stack, id
|
||||
switch stack {
|
||||
case "notheld":
|
||||
return "", fmt.Errorf("%w: notheld has no hold — nothing was changed", ErrActionRefused)
|
||||
case "broken":
|
||||
return "", errors.New("saving settings: disk full")
|
||||
}
|
||||
return "the hold on " + stack + " is cleared (it was: held after a failed update)", nil
|
||||
}}
|
||||
o := NewOperatorActions(context.Background(), h, nil)
|
||||
o.Reconcile([]OperatorAction{{ID: 7, Action: OpClearHold, Arg: "kimai"}})
|
||||
if gotStack != "kimai" || gotID != 7 {
|
||||
t.Fatalf("handler called with (%q, %d), want (kimai, 7)", gotStack, gotID)
|
||||
}
|
||||
if r := resultFor(t, o, 7); r.Outcome != OutcomeDone || r.Message != "the hold on kimai is cleared (it was: held after a failed update)" {
|
||||
t.Fatalf("result %+v", r)
|
||||
}
|
||||
o.Reconcile([]OperatorAction{{ID: 8, Action: OpClearHold, Arg: "notheld"}, {ID: 9, Action: OpClearHold, Arg: "broken"}})
|
||||
if r := resultFor(t, o, 8); r.Outcome != OutcomeRefused || !strings.Contains(r.Message, "has no hold") {
|
||||
t.Errorf("not held: %+v", r)
|
||||
}
|
||||
if r := resultFor(t, o, 9); r.Outcome != OutcomeFailed {
|
||||
t.Errorf("a save error must be failed, not refused: %+v", r)
|
||||
}
|
||||
// nil handler → refused, not a crash
|
||||
o2 := NewOperatorActions(context.Background(), OperatorActionHandlers{}, nil)
|
||||
o2.Reconcile([]OperatorAction{{ID: 1, Action: OpClearHold, Arg: "kimai"}})
|
||||
if r := resultFor(t, o2, 1); r.Outcome != OutcomeRefused {
|
||||
t.Errorf("nil handler: %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -1982,8 +1982,36 @@ type RestoreHold struct {
|
||||
// first stop or a repeat within 24 h (Trip >= 2 → the sentence says support is informed).
|
||||
UnhealthyKind string `json:"unhealthy_kind,omitempty"`
|
||||
Trip int `json:"trip,omitempty"`
|
||||
|
||||
// R-1340 (2026-10-29): what put a restore hold in place, so the operator's „HELD STOPPED" mail tells the real
|
||||
// story instead of the option-C database story for every cause. All empty on a hold written before them: the mail
|
||||
// then names only what the record carries (cmd/controller heldStoppedOperatorText).
|
||||
//
|
||||
// RestoreKind is which restore: RestoreKindOffsite or RestoreKindLocal.
|
||||
RestoreKind string `json:"restore_kind,omitempty"`
|
||||
// MixedCause (restore_mixed only) is the branch that held the app: MixedCause*.
|
||||
MixedCause string `json:"mixed_cause,omitempty"`
|
||||
// FailedStep is the put-back step that failed, with its error (MixedCausePutBack).
|
||||
FailedStep string `json:"failed_step,omitempty"`
|
||||
// DBReplay: the restore's own failure was its database replay. False = the app's database was never touched
|
||||
// by the failure, so the mail never says „database".
|
||||
DBReplay bool `json:"db_replay,omitempty"`
|
||||
}
|
||||
|
||||
// Restore kinds and mixed-hold causes (R-1340). See RestoreHold.RestoreKind / MixedCause.
|
||||
const (
|
||||
RestoreKindOffsite = "offsite"
|
||||
RestoreKindLocal = "local"
|
||||
|
||||
// MixedCauseDefinitionWrite: the backup's definition could not be written (no data moved yet) and writing the
|
||||
// live definition back failed too (R-1256 H4).
|
||||
MixedCauseDefinitionWrite = "definition_write"
|
||||
// MixedCausePutBack: the restore failed and putting the live data back failed at FailedStep (D20/R-1256 H5).
|
||||
MixedCausePutBack = "put_back"
|
||||
// MixedCauseInterruptedPutBack: a restore cut by a stop could not be put back at the next start (R-1251 H2).
|
||||
MixedCauseInterruptedPutBack = "interrupted_put_back"
|
||||
)
|
||||
|
||||
// Hold reasons. See RestoreHold.Reason.
|
||||
const (
|
||||
HoldReasonRestoreFailed = "" // R-379/R-380: a restore AND its rollback failed
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
)
|
||||
|
||||
// D64 = B / R-1343: when the operator clears a hold, the ONE held flag kept outside the hold store — „the last update
|
||||
// ended held" — goes too. Left set, the crash-loop observer and the nightly update leg kept skipping an app whose hold
|
||||
// was gone, until the controller restarted.
|
||||
// RED-PROOF (2026-10-29): with ForgetUpdateHeld's body reduced to `return false`, this fails on „a held update must be forgotten".
|
||||
func TestD64_ForgetUpdateHeld(t *testing.T) {
|
||||
m := &Manager{cfg: &config.Config{}, logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
|
||||
"kimai": {Name: "kimai", Deployed: true},
|
||||
"running": {Name: "running", Deployed: true, Updating: true},
|
||||
}}
|
||||
m.finishUpdateKey("kimai", UpdatePhaseFailed, UpdateErrorKeyHeld, "a kimai leállítva marad")
|
||||
m.markUpdateHeld("kimai")
|
||||
m.markUpdateHeld("running")
|
||||
|
||||
if !m.ForgetUpdateHeld("kimai") {
|
||||
t.Fatal("a held update must be forgotten")
|
||||
}
|
||||
st, _ := m.GetStack("kimai")
|
||||
if st.updateHeld || st.UpdateErrorKey != "" || st.UpdateError != "" || st.UpdatePhase != "" {
|
||||
t.Fatalf("held state left: held=%v key=%q err=%q phase=%q", st.updateHeld, st.UpdateErrorKey, st.UpdateError, st.UpdatePhase)
|
||||
}
|
||||
// The two readers of the flag (ObserveUnhealthy's skip, unhealthy.go; legCandidate's LegSkipHeld, unattended.go)
|
||||
// read the stored stack — not called here, because ObserveUnhealthy reaches docker (R-650).
|
||||
m.mu.RLock()
|
||||
skipped := m.stacks["kimai"].updateHeld || m.stacks["kimai"].UpdateErrorKey == UpdateErrorKeyHeld
|
||||
m.mu.RUnlock()
|
||||
if skipped {
|
||||
t.Fatal("the observer and the update leg still skip kimai")
|
||||
}
|
||||
// A running update is never touched; an app with nothing held reports false.
|
||||
if m.ForgetUpdateHeld("running") {
|
||||
t.Fatal("a running update must not be touched")
|
||||
}
|
||||
if m.ForgetUpdateHeld("kimai") || m.ForgetUpdateHeld("nosuch") {
|
||||
t.Fatal("nothing held → false")
|
||||
}
|
||||
}
|
||||
@@ -209,6 +209,25 @@ func (m *Manager) ClearUpdateState(name string) {
|
||||
s.HealthProbe = nil
|
||||
}
|
||||
|
||||
// ForgetUpdateHeld (D64 = B, R-1343) forgets that the app's last update ended HELD, once the hold itself is cleared
|
||||
// by the operator. updateHeld is the ONE held flag kept outside the hold store: it makes the crash-loop observer
|
||||
// (ObserveUnhealthy) and the nightly update leg (legCandidate) skip the app, and keeps the hold's sentence as the
|
||||
// update error. Left set, an app whose hold was cleared would stay unobserved and never be updated by night until
|
||||
// the controller restarts. A running update is not touched. Returns whether anything was forgotten.
|
||||
// Pinned by TestD64_ForgetUpdateHeld (this package) and TestD64_ClearHoldForgetsTheUpdateHeldFlag (cmd/controller).
|
||||
func (m *Manager) ForgetUpdateHeld(name string) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
s, ok := m.stacks[name]
|
||||
if !ok || s.Updating || (!s.updateHeld && s.UpdateErrorKey != UpdateErrorKeyHeld) {
|
||||
return false
|
||||
}
|
||||
s.updateHeld = false
|
||||
s.UpdatePhase, s.UpdatePhaseLabel = "", ""
|
||||
s.UpdateError, s.UpdateErrorKey, s.UpdateErrorArgs = "", "", nil
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *Manager) markUpdateHeld(name string) {
|
||||
m.mu.Lock()
|
||||
if s, ok := m.stacks[name]; ok {
|
||||
|
||||
Reference in new issue
Block a user