clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type. - --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart). - R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause, failed put-back step, database replay or not, kept copy) — never the option-C database story by default. - R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
1 parent
c7bb776e00
commit
5b3b541602
25 files changed
+1809
-83
No files matched your search
@@ -119,7 +119,8 @@ func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
|
||||
return false
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
|
||||
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
|
||||
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir,
|
||||
MixedCause: settings.MixedCauseInterruptedPutBack, FailedStep: "putting the interrupted restore's copy back failed: " + cause.Error()}
|
||||
if err := g.holds.SetRestoreHold(h); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── D64 = B (operator ruling 2026-10-29 12:10) + R-1343: the operator clears a hold IN THE RUNNING CONTROLLER ──────
|
||||
//
|
||||
// One door for both of the operator's routes — the hub's „Clear hold" (operator action clear_hold) and the box command
|
||||
// `--clear-restore-hold`, which now asks the running controller over its local socket (cmd/controller/holdctl.go).
|
||||
//
|
||||
// WHY IN THIS PROCESS (R-1343, measured 2026-10-29 on Tester 1): the CLI used to run as a SECOND process that edited
|
||||
// settings.json while the running controller kept the hold in its own in-memory Settings — and that controller's next
|
||||
// save (after a hub report) wrote the hold back. Clearing through the running Settings removes the hold from memory
|
||||
// AND saves the file at once, so no later save of any component can bring it back.
|
||||
// Pinned by TestD64_ClearHoldStaysGoneAfterALaterSave (cmd/controller).
|
||||
//
|
||||
// EVERY REASON (D64 = B): the hold's reason does not matter — R-379 now reads „a person — on the box, or the operator
|
||||
// on the hub". One reason needs a second act: a put-back WAITING for its drive (HoldReasonPutBackPending) is also
|
||||
// recorded in the app-stop marker, and a controller start with the drive bound would pour that old copy back over
|
||||
// whatever the app wrote after the clear. So the clear also drops the app from the marker; the copy stays on disk
|
||||
// (nothing is deleted). It runs under resumeMu, so it can never interleave with a put-back of the same app.
|
||||
// Pinned by TestD64_ClearPutBackPendingDropsTheMarkerEntry.
|
||||
|
||||
// ErrHoldBadStack: the argument is not a valid stack name. ErrNotHeld: the app has no hold. Both are refusals —
|
||||
// nothing was changed.
|
||||
var (
|
||||
ErrHoldBadStack = errors.New("not a valid app name")
|
||||
ErrNotHeld = errors.New("this app has no hold")
|
||||
)
|
||||
|
||||
// IsValidHoldStack reports whether s may name an app for the hold commands (the backup package's strict
|
||||
// isSafeStackName: letters, digits, '-' and '_', at most 64).
|
||||
func IsValidHoldStack(s string) bool { return isSafeStackName(s) }
|
||||
|
||||
// ClearHoldByOperator lifts stack's hold, whatever its reason, in the hold store the running controller uses, and
|
||||
// returns the hold it lifted. A waiting put-back is dropped from the app-stop marker too (see above). Refusals
|
||||
// (ErrHoldBadStack, ErrNotHeld) change nothing.
|
||||
func (g *AppStopGuard) ClearHoldByOperator(stack string) (settings.RestoreHold, error) {
|
||||
if !isSafeStackName(stack) {
|
||||
return settings.RestoreHold{}, ErrHoldBadStack
|
||||
}
|
||||
if g == nil || g.holds == nil {
|
||||
return settings.RestoreHold{}, errors.New("no hold store is wired on this box")
|
||||
}
|
||||
resumeMu.Lock()
|
||||
defer resumeMu.Unlock()
|
||||
h, ok := g.holds.GetRestoreHold(stack)
|
||||
if !ok {
|
||||
return settings.RestoreHold{}, ErrNotHeld
|
||||
}
|
||||
cleared, err := g.holds.ClearRestoreHold(stack)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
if !cleared {
|
||||
return settings.RestoreHold{}, ErrNotHeld
|
||||
}
|
||||
if h.Reason == settings.HoldReasonPutBackPending {
|
||||
g.dropFromMarker(stack)
|
||||
g.logger.Printf("[WARN] [appstop] %s: its waiting put-back is DROPPED with the hold — the copy stays on disk at %s and is never poured back by itself", stack, h.SafetyDump)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -425,9 +425,12 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
|
||||
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
|
||||
return
|
||||
}
|
||||
// R-1340: this branch is only ever an off-site restore's database replay (offbox_reconstitute's rollback).
|
||||
h := settings.RestoreHold{
|
||||
Stack: stack,
|
||||
At: time.Now().UTC().Format(time.RFC3339),
|
||||
Stack: stack,
|
||||
At: time.Now().UTC().Format(time.RFC3339),
|
||||
RestoreKind: settings.RestoreKindOffsite,
|
||||
DBReplay: true,
|
||||
}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
@@ -447,6 +450,15 @@ func (m *Manager) holdAppAfterFailedRollback(stack string, replayErr, rollbackEr
|
||||
}
|
||||
}
|
||||
|
||||
// mixedHoldWhy (R-1340) is what a restore_mixed hold records about its cause (settings.RestoreHold's R-1340 fields).
|
||||
type mixedHoldWhy struct {
|
||||
kind string // settings.RestoreKind*
|
||||
cause string // settings.MixedCause*
|
||||
step string // the step that failed, with its error
|
||||
dbReplay bool // the restore's own failure was its database replay
|
||||
copyDir string // where the copy of the live volumes is kept; "" = none kept
|
||||
}
|
||||
|
||||
// liveDefinition is the app's definition as it ran before an off-site restore wrote the snapshot's
|
||||
// (R-893): a copy of its compose files in a temporary folder, and its full env held in memory only.
|
||||
type liveDefinition struct {
|
||||
@@ -511,7 +523,11 @@ func (m *Manager) captureLiveDefinition(stack string) (*liveDefinition, error) {
|
||||
// written back (when the snapshot's was written), the database service is stopped again, and the app is
|
||||
// held with HoldReasonRestoreMixed. The operator is notified with no rollback error — that is how the
|
||||
// notification tells this case from R-379's double failure.
|
||||
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition) {
|
||||
//
|
||||
// why (R-1340) is what the hold records about its cause, so the operator's mail tells the real story — this hold is
|
||||
// written by three different branches now (a definition write, a put-back that failed at one of its steps), not
|
||||
// only by option C's database replay.
|
||||
func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *liveDefinition, why mixedHoldWhy) {
|
||||
// The hold is PERSISTED FIRST (security review 2026-10-08): a controller that dies between the stop and
|
||||
// the hold would otherwise restart the app from the R-166 marker with nothing refusing it. When the hold
|
||||
// cannot be persisted, the app-stop marker is KEPT, so nothing reads the window as finished.
|
||||
@@ -519,7 +535,8 @@ func (m *Manager) holdAppAfterMixedRestore(stack string, replayErr error, live *
|
||||
if m.settings == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: cannot persist the restore hold — no settings wired; the app is stopped but NOTHING will refuse a restart", stack)
|
||||
} else {
|
||||
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}
|
||||
h := settings.RestoreHold{Stack: stack, At: time.Now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
|
||||
RestoreKind: why.kind, MixedCause: why.cause, FailedStep: why.step, DBReplay: why.dbReplay, SafetyDump: why.copyDir}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
}
|
||||
@@ -557,7 +574,8 @@ func (m *Manager) restartAtLiveDefinition(stack string, live *liveDefinition, ca
|
||||
if live != nil {
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: the definition write failed (%v) and writing the live definition back failed too: %v — holding the app stopped", stack, cause, err)
|
||||
m.holdAppAfterMixedRestore(stack, cause, nil)
|
||||
m.holdAppAfterMixedRestore(stack, cause, nil, mixedHoldWhy{kind: settings.RestoreKindOffsite, cause: settings.MixedCauseDefinitionWrite,
|
||||
step: "writing the live app definition back failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] %s: the definition write failed (%v); the live definition is written back before the start", stack, cause)
|
||||
@@ -1060,7 +1078,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// R-893 option A: no volume has moved yet, so only the live definition is written back and the app starts.
|
||||
// The failing copy may have placed part of its files, so the sentence names placed files (+1).
|
||||
if definitionMoved {
|
||||
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, cErr, liveDef, undo, false, res.FilesPlaced+1, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Nothing moved yet: best-effort bring-up — a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1095,7 +1113,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// R-893 option A: the replay removes a volume BEFORE it recreates it, so even nVols == 0 may have lost one —
|
||||
// with a copy, it is always poured back; with no copy the old rule decides (and holds when one moved).
|
||||
if definitionMoved || nVols > 0 || undo != nil {
|
||||
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, vErr, liveDef, undo, nVols > 0 || undo != nil, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Nothing replaced yet: bring the app back up rather than leaving an outage, then surface it.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1114,7 +1132,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
if err := m.stackProvider.StartStackServices(stack, dbServices); err != nil {
|
||||
// R-893 option C (security review 2026-10-08): volumes or the definition already moved → HOLD.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
// Best-effort bring-up: a failed restore must not also be an outage.
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
@@ -1166,7 +1184,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
// under them — the copy of the live volumes is poured back over the rolled-back database, the live
|
||||
// definition written back, and the app starts exactly as it was. A put-back that fails holds, as before.
|
||||
if definitionMoved || res.VolumesReplayed > 0 {
|
||||
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed")
|
||||
return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, "err.backup.db_restore_failed_held_mixed", settings.RestoreKindOffsite)
|
||||
}
|
||||
if sErr := restartStack(); sErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] %s: start after a successful rollback failed: %v", stack, sErr)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
@@ -219,11 +220,23 @@ func (m *Manager) putBack(stack string, c *liveVolumeCopy) error {
|
||||
// failure's own held sentence) and the copy stays on disk. volumesMoved with no copy is such a case: nothing can
|
||||
// put them back. filesPlaced chooses the household's sentence: placed files are not taken back (as ruled for
|
||||
// option C, a file mix does not hold), so the sentence then says so instead of „nothing changed".
|
||||
//
|
||||
// restoreKind (R-1340) is settings.RestoreKindOffsite or RestoreKindLocal: the hold records it, with the step that
|
||||
// failed and whether the restore's failure was a database replay (holdKey says so — it is the callers' own choice of
|
||||
// the household's sentence), so the operator's mail names the real cause.
|
||||
func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liveDefinition, c *liveVolumeCopy, volumesMoved bool,
|
||||
filesPlaced int, restart func() error, holdKey string) error {
|
||||
filesPlaced int, restart func() error, holdKey, restoreKind string) error {
|
||||
why := func(step string) mixedHoldWhy {
|
||||
w := mixedHoldWhy{kind: restoreKind, cause: settings.MixedCausePutBack, step: step,
|
||||
dbReplay: holdKey == "err.backup.db_restore_failed_held_mixed"}
|
||||
if c != nil && c.keep {
|
||||
w.copyDir = c.dir
|
||||
}
|
||||
return w
|
||||
}
|
||||
if volumesMoved && c == nil {
|
||||
m.logger.Printf("[ERROR] [offbox] %s: volumes were replaced and there is no copy to put back — holding the app", stack)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("the restore had already changed the app's data and there was no copy of the live volumes to put back"))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
// R-1256 H5: a put-back under an app that could not be stopped could mix the copy with what the app writes.
|
||||
@@ -231,14 +244,14 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.StopStack(stack); err != nil {
|
||||
c.keepIfAny()
|
||||
m.logger.Printf("[ERROR] [offbox] %s: stopping the app before the put-back FAILED: %v — nothing is put back; the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("stopping the app before the put-back failed, so nothing was put back: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
if volumesMoved {
|
||||
if err := m.putBack(stack, c); err != nil {
|
||||
c.keep = true
|
||||
m.logger.Printf("[ERROR] [offbox] %s: putting the live volumes back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dir)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("putting the live volumes back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
@@ -246,7 +259,7 @@ func (m *Manager) putBackAfterFailedRestore(stack string, cause error, live *liv
|
||||
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stack, live.dir, live.env); err != nil {
|
||||
c.keepIfAny() // R-1256 H5 (D20): a put-back that fails keeps its copy for support
|
||||
m.logger.Printf("[ERROR] [offbox] %s: writing the live definition back FAILED: %v — the copy stays at %s; holding the app", stack, err, c.dirOrNone())
|
||||
m.holdAppAfterMixedRestore(stack, cause, live)
|
||||
m.holdAppAfterMixedRestore(stack, cause, live, why("writing the live app definition back failed: "+err.Error()))
|
||||
return util.MsgError(holdKey, stack)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed,
|
||||
// whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail
|
||||
// (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on
|
||||
// Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its
|
||||
// DATABASE restore failed and was rolled back.
|
||||
//
|
||||
// RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from
|
||||
// holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ /
|
||||
// „kind """.
|
||||
|
||||
func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold {
|
||||
t.Helper()
|
||||
if m.settings == nil {
|
||||
t.Fatal("fixture: no settings wired")
|
||||
}
|
||||
h, ok := m.settings.GetRestoreHold(stack)
|
||||
if !ok {
|
||||
t.Fatalf("%s is not held", stack)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails.
|
||||
func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) {
|
||||
m, vp, rig, _, notified, _ := r893PutBackFixture(t)
|
||||
rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use")
|
||||
p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's
|
||||
m.SetStackProvider(p)
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed restore must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack {
|
||||
t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") {
|
||||
t.Errorf("failed step %q — want the stop and its error", h.FailedStep)
|
||||
}
|
||||
if h.DBReplay {
|
||||
t.Error("a volume failure must not be recorded as a database replay")
|
||||
}
|
||||
if h.RestoreKind != settings.RestoreKindOffsite {
|
||||
t.Errorf("restore kind %q, want offsite", h.RestoreKind)
|
||||
}
|
||||
if h.SafetyDump == "" {
|
||||
t.Error("the kept copy's folder must be recorded")
|
||||
} else if _, err := os.Stat(h.SafetyDump); err != nil {
|
||||
t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err)
|
||||
}
|
||||
if *notified != 1 {
|
||||
t.Errorf("hold notices %d, want 1", *notified)
|
||||
}
|
||||
}
|
||||
|
||||
// The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded.
|
||||
func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) {
|
||||
m, vp, _, _, _, _ := r893PutBackFixture(t)
|
||||
m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2})
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed restore must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay {
|
||||
t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved.
|
||||
func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) {
|
||||
m, _, _ := r1256DefFailFixture(t, 2)
|
||||
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil {
|
||||
t.Fatal("a failed definition write must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "immich")
|
||||
if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay {
|
||||
t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "writing the live app definition back failed") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
|
||||
// A LOCAL restore (from the app's own backup) records itself as local, never off-site.
|
||||
func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) {
|
||||
m, _, rig, _ := r1177Fixture(t)
|
||||
if m.settings == nil {
|
||||
sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.settings = sett
|
||||
}
|
||||
rig.unitErr = errors.New("tar: short read")
|
||||
rig.putBackErr = errors.New("tar xf failed")
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err == nil {
|
||||
t.Fatal("a failed put-back must be surfaced")
|
||||
}
|
||||
h := holdOf(t, m, "app")
|
||||
if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay {
|
||||
t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
|
||||
// A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause.
|
||||
func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.putErr = errors.New("compose down: timeout")
|
||||
r.guard().Recover()
|
||||
h, ok := r.sett.GetRestoreHold("immich")
|
||||
if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack {
|
||||
t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok)
|
||||
}
|
||||
if !strings.Contains(h.FailedStep, "compose down: timeout") {
|
||||
t.Errorf("failed step %q", h.FailedStep)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package backup
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -530,7 +531,7 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
|
||||
// without one (the unit holds no volume) only a database replay that began can have changed data — and with no
|
||||
// copy to pour back that is the R-379 half state: held, never started (operator ruling 2026-08-22).
|
||||
putBack := func(cause error, holdKey string, moved bool) (UnitRestoreResult, error) {
|
||||
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey)
|
||||
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey, settings.RestoreKindLocal)
|
||||
}
|
||||
// R-353: the count is captured even when the replay errors — a partial replay is a fact the
|
||||
// customer's sentence has to be built from, and discarding it on the error path is how Scenario C
|
||||
|
||||
Reference in new issue
Block a user