D22: every restore puts the data back exactly on failure and after a power cut (R-1177, R-1195); whole restore asks the unit first (R-1178); no mirror during a restore (R-1180); W22 restore refusals in Hungarian; unreleased
gates / gates (push) Successful in 1m40s

This commit is contained in:
2026-10-21 21:19:46 +02:00
parent 7848d4aaac
commit d582feb245
25 changed files with 1065 additions and 71 deletions
@@ -46,6 +46,8 @@ const (
ReasonOffboxReconstitute AppStopReason = "offbox_reconstitute"
// ReasonAppExport — a .fab export taken with "stop the app first".
ReasonAppExport AppStopReason = "app_export"
// ReasonUnitRestore — a local restore from a recovery unit (the app's own, the second drive's, a kept one).
ReasonUnitRestore AppStopReason = "unit_restore"
)
// humanReason is the operator-facing phrasing for each reason.
@@ -57,6 +59,8 @@ func (r AppStopReason) humanReason() string {
return "an off-site restore"
case ReasonAppExport:
return "an app export"
case ReasonUnitRestore:
return "a local restore"
default:
return string(r)
}
@@ -74,6 +78,8 @@ func (r AppStopReason) MessageKey() string {
return "appstop.reason.offbox_reconstitute"
case ReasonAppExport:
return "appstop.reason.app_export"
case ReasonUnitRestore:
return "appstop.reason.unit_restore"
default:
return ""
}
@@ -87,6 +93,9 @@ type AppStopMarker struct {
Reason AppStopReason `json:"reason"`
Stacks []string `json:"stacks"`
StartedAt time.Time `json:"started_at"`
// Undo (D22, R-1195) names, per app, the copy of its live volumes and definition a restore took right after the
// stop (restore_crash_putback.go). Set only once the copy is whole; Recover pours it back before the restart.
Undo map[string]string `json:"undo,omitempty"`
}
// AppStopStarter is the one thing recovery needs: the ability to start a stack. StartStack must be
@@ -121,6 +130,9 @@ type AppStopGuard struct {
now func() time.Time
// starter is only needed by Recover; Begin/End work without one.
starter AppStopStarter
// putBacker (D22, R-1195) pours a restore's undo copy back before Recover restarts the app. nil = no put-back
// (the pre-D22 restart, unchanged).
putBacker InterruptedRestorePutBacker
// suppressed (R-330) is the in-memory "the alarm must not fire for these, we are holding them"
// set. It is deliberately NOT persisted: after a restart the guard no longer holds anything —
@@ -147,6 +159,9 @@ type AppStopRecovery struct {
// (R-174). The marker is kept for these too, but they are NOT a fault and MUST NOT alarm: the
// holder owns the restart. Separate from Failed for the reason recorded on ErrStartRefused.
Refused []string
// PutBack are apps whose interrupted restore was undone from its copy before the restart (D22, R-1195): their
// data is as it was before the restore.
PutBack []string
}
// Alarming reports whether this recovery is worth paging an operator about. A recovery that only
@@ -197,6 +212,9 @@ func (r *AppStopRecovery) Detail() string {
if len(r.Refused) > 0 {
d += fmt.Sprintf(" held_by_drive=%v", r.Refused)
}
if len(r.PutBack) > 0 {
d += fmt.Sprintf(" put_back=%v", r.PutBack)
}
return d
}
@@ -218,6 +236,43 @@ func (g *AppStopGuard) SetStarter(s AppStopStarter) {
g.starter = s
}
// InterruptedRestorePutBacker pours an interrupted restore's undo copy back (restore_crash_putback.go). It must leave
// the app stopped; Recover starts it.
type InterruptedRestorePutBacker interface {
PutBackInterruptedRestore(stack, undoDir string) error
}
// SetPutBacker wires the put-back Recover runs for an app whose restore was cut after its copy was taken.
func (g *AppStopGuard) SetPutBacker(p InterruptedRestorePutBacker) {
if g != nil {
g.putBacker = p
}
}
// SetUndo records, in the active marker, where stack's undo copy is (D22, R-1195). Called once the copy is whole;
// a marker that is not active (or not for this app) is left alone.
func (g *AppStopGuard) SetUndo(stack, dir string) error {
if g == nil || g.path == "" {
return nil
}
m, ok := g.read()
if !ok || !m.Active {
return fmt.Errorf("no active app-stop marker to record the undo copy in")
}
found := false
for _, s := range m.Stacks {
found = found || s == stack
}
if !found {
return fmt.Errorf("the active app-stop marker does not hold %s", stack)
}
if m.Undo == nil {
m.Undo = map[string]string{}
}
m.Undo[stack] = dir
return g.write(m)
}
// Begin records that `stacks` are about to be stopped by `reason`. It MUST be called BEFORE the
// first stop — an error here means the marker could not be written, and the caller must not proceed
// to stop an app it cannot promise to restart.
@@ -304,6 +359,21 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
continue
}
if dir := m.Undo[name]; dir != "" && g.putBacker != nil {
// D22 (R-1195): the restore was cut after its copy of the live data was taken — the app may be on the
// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery
// (the marker is kept, so the next startup tries again) and the app is not started on unknown data.
if _, statErr := os.Stat(dir); statErr != nil {
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
} else if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
res.Failed = append(res.Failed, name)
continue
} else {
g.logger.Printf("[WARN] [appstop] crash recovery: %s's interrupted restore is undone — its data is as before the restore (R-1195)", name)
res.PutBack = append(res.PutBack, name)
}
}
if err := g.starter.StartStack(name); err != nil {
// R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be
// started (an absent data drive), so the app is left down deliberately and the holder
@@ -324,6 +394,7 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
sort.Strings(res.Failed)
sort.Strings(res.Refused)
sort.Strings(res.Restarted)
sort.Strings(res.PutBack)
// The marker is kept for BOTH unfinished outcomes, for the same reason and with different
// urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished
@@ -974,6 +974,7 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
}
undo = c
defer undo.drop(m, stack)
m.persistUndoForCrash(stack, undo) // R-1195: from here a power cut puts the app back on start
}
// v0.275.0: the snapshot's own definition, written while nothing of the app's data has been touched
// yet — a failure here restarts the app as it was.
@@ -134,6 +134,15 @@ func (m *Manager) planLiveVolumeCopy(vols []string) (need int64, exists map[stri
// copyLiveVolumes tars every existing volume of the plan into a fresh restoreUndoDir. Runs with the app
// STOPPED, so each copy is consistent. On any error the partial copy is removed and nothing has moved yet.
func (m *Manager) copyLiveVolumes(stack, nsRoot string, vols []string, exists map[string]bool) (*liveVolumeCopy, error) {
// D22: when none of the volumes exists there is nothing to copy — the copy is the list of volumes the backup adds
// (removed on a put-back), with no folder.
none := true
for _, v := range vols {
none = none && !exists[v]
}
if none {
return &liveVolumeCopy{absent: append([]string(nil), vols...)}, nil
}
dir := filepath.Join(restoreUndoDir(nsRoot, stack), time.Now().UTC().Format("20060102T150405Z"))
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
+3
View File
@@ -19,6 +19,9 @@ type RestoreOpResult struct {
// Interrupted marks a restore that was still in flight when the controller stopped (R-550): found
// at the next start, recorded as a failure with RestoreInterruptedMessage.
Interrupted bool `json:"interrupted,omitempty"`
// PutBack marks an interrupted restore that the start-up put the data back for (D22, R-1195): the message then
// says the app's data is as it was before the restore, not „start it again".
PutBack bool `json:"put_back,omitempty"`
}
// RestoreResultWindow bounds how long a finished restore still counts as "what just happened".
@@ -2,6 +2,7 @@ package backup
import (
"context"
"errors"
"io"
"log"
"os"
@@ -213,7 +214,7 @@ func TestR102_Tier2UnitRestoreTakesTheSingleWriterFlag(t *testing.T) {
if err == nil {
t.Fatal("a second restore started while one was in flight")
}
if !strings.Contains(err.Error(), "already in progress") {
if !errors.Is(err, ErrRestoreBusy) {
t.Errorf("refusal = %v, want the single-writer refusal", err)
}
if f.fake.stopped {
@@ -0,0 +1,391 @@
package backup
import (
"context"
"errors"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// D22 (operator ruling 2026-10-21 20:05): the off-site restore's „put back exactly as it was" (R-893 option A) now
// covers the LOCAL unit restore (R-1177) — and a restore cut by a power loss is put back on start (R-1195,
// restore_crash_putback.go). These tests pin the CONSEQUENCE: which bytes are poured back, what is removed, which
// definition is written last, whether the app runs, what stays on disk.
const r1177Vol, r1177NewVol = "app_data", "app_newvol"
type r1177Rig struct {
unitVolDir string
undoRoot string
poured map[string]string
removed []string
unitErr error // the unit's own volume replay
putBackErr error
defsWrote []string // RecreateStackDefinitionFromUnit's folders, in order
liveDir string
}
// r1177Provider records which definition folder was written, so a test can tell the unit's from the live one.
type r1177Provider struct {
*fakeRecoveryProvider
rig *r1177Rig
}
func (p *r1177Provider) RecreateStackDefinitionFromUnit(name, dir string, env map[string]string) error {
p.rig.defsWrote = append(p.rig.defsWrote, dir)
return p.fakeRecoveryProvider.RecreateStackDefinitionFromUnit(name, dir, env)
}
// r1177Fixture: the r47 unit for "app" (Postgres dump), plus a volume archive in the unit, a live definition on disk,
// one live volume the box has (app_data) and one only the unit has (app_newvol). Docker is never reached.
func r1177Fixture(t *testing.T) (*Manager, *r1177Provider, *r1177Rig, *[]string) {
t.Helper()
m, prov, imported := r47UnitFixture(t, immichLikeCompose, true)
rig := &r1177Rig{poured: map[string]string{}, liveDir: t.TempDir()}
prov.info.StackDir = rig.liveDir
mustWrite(t, filepath.Join(rig.liveDir, "docker-compose.yml"), "LIVE compose\n")
mustWrite(t, filepath.Join(rig.liveDir, "app.yaml"), "deployed: true\nenv:\n SUBDOMAIN: app\n")
p := &r1177Provider{fakeRecoveryProvider: prov, rig: rig}
m.stackProvider = p
unit := m.primaryUnitDirFor("app")
rig.unitVolDir = UnitVolumeDumpDir(unit)
mustWrite(t, filepath.Join(rig.unitVolDir, r1177Vol+".tar"), "UNIT bytes")
mustWrite(t, filepath.Join(rig.unitVolDir, r1177NewVol+".tar"), "UNIT new")
rig.undoRoot = restoreUndoDir(m.namespaceRoot(m.GetAppDrivePath("app")), "app")
m.undoVolumeProbeFn = func(vol string) (int64, bool, error) { return 1000, vol == r1177Vol, nil }
m.undoVolumeTarFn = func(vol, dir string) error {
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("LIVE bytes of "+vol), 0o600)
}
m.undoVolumeRemoveFn = func(vol string) error { rig.removed = append(rig.removed, vol); return nil }
m.volumeReplayFrom = func(_ string, dir string) (int, error) {
if dir == rig.unitVolDir {
if rig.unitErr != nil {
return 0, rig.unitErr
}
return 2, nil
}
if !strings.HasPrefix(dir, rig.undoRoot) {
t.Errorf("a volume replay from an unexpected folder %q", dir)
return 0, errors.New("unexpected folder")
}
if rig.putBackErr != nil {
return 0, rig.putBackErr
}
entries, _ := os.ReadDir(dir)
n := 0
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".tar") {
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
rig.poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
n++
}
}
return n, nil
}
return m, p, rig, imported
}
// assertPutBack is the consequence D22 promises: the live bytes are back, the unit's extra volume is gone, the LIVE
// definition was written last, the app runs, the copy is removed, and the household reads „nothing changed".
func assertPutBack(t *testing.T, err error, p *r1177Provider, rig *r1177Rig) {
t.Helper()
if want := util.MsgError("err.backup.restore_failed_put_back", "app").Error(); err == nil || err.Error() != want {
t.Fatalf("the household must read the put-back sentence, got: %v", err)
}
if rig.poured[r1177Vol] != "LIVE bytes of "+r1177Vol {
t.Fatalf("the live volume was not poured back byte for byte: %v", rig.poured)
}
if len(rig.removed) != 1 || rig.removed[0] != r1177NewVol {
t.Fatalf("the volume only the unit has must be removed, removed %v", rig.removed)
}
if n := len(rig.defsWrote); n == 0 || !strings.Contains(rig.defsWrote[n-1], "felhom-livedef-") {
t.Fatalf("the LIVE definition must be written back last, wrote %v", rig.defsWrote)
}
if !p.fullStarted || p.calls[len(p.calls)-1] != "start" {
t.Fatalf("the app must run again, calls %v", p.calls)
}
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
t.Fatalf("the copy must be removed once it is poured back, found %v", left)
}
}
// R-1177's measured case (C3, 9202): a damaged volume archive in the unit. Before D22 the live volume was gone, the
// database replay skipped and the app started on nothing (Postgres crash-looped).
// RED-PROOF: on the pre-D22 restore_unit.go this fails on the error text („completed with data errors") and on the
// live bytes never poured back.
func TestR1177_ADamagedVolumePutsTheLiveDataBackAndStartsTheApp(t *testing.T) {
m, p, rig, imported := r1177Fixture(t)
rig.unitErr = errors.New("failed to restore 1 volume(s): [app_data]")
_, err := m.RestoreFromRecoveryUnit("app")
assertPutBack(t, err, p, rig)
if len(*imported) != 0 {
t.Fatalf("the database copy was replayed over a volume that failed (R-638): %v", *imported)
}
}
// A database replay that fails half-way: the live volumes (the database's own among them) go back.
func TestR1177_AFailedDatabaseReplayPutsTheLiveDataBack(t *testing.T) {
m, p, rig, _ := r1177Fixture(t)
m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("ERROR: relation exists") }
_, err := m.RestoreFromRecoveryUnit("app")
assertPutBack(t, err, p, rig)
}
// A put-back that fails: the app is HELD and the copy stays on disk — never started on data nobody knows.
func TestR1177_AFailedPutBackHoldsTheAppAndKeepsTheCopy(t *testing.T) {
m, p, rig, _ := r1177Fixture(t)
rig.unitErr = errors.New("tar: short read")
rig.putBackErr = errors.New("tar xf failed")
_, err := m.RestoreFromRecoveryUnit("app")
if want := util.MsgError("err.backup.restore_failed_held_mixed", "app").Error(); err == nil || err.Error() != want {
t.Fatalf("a failed put-back must hold, got: %v", err)
}
if p.fullStarted {
t.Fatalf("the app must NOT start on volumes that could not be put back — calls %v", p.calls)
}
if left := undoCopies(t, rig.undoRoot); len(left) != 1 || filepath.Base(left[0]) != r1177Vol+".tar" {
t.Fatalf("the copy must stay on disk for support, found %v", left)
}
}
// Not enough room for the copy: refused BEFORE the stop, nothing touched.
func TestR1177_NoRoomForTheCopyRefusesBeforeTheStop(t *testing.T) {
m, p, rig, _ := r1177Fixture(t)
m.freeBytesFn = func(string) int64 { return 10 }
_, err := m.RestoreFromRecoveryUnit("app")
if err == nil || !strings.Contains(err.Error(), "Nincs elég szabad hely") {
t.Fatalf("want the no-room refusal, got %v", err)
}
if len(p.calls) != 0 || len(rig.poured) != 0 {
t.Fatalf("a refusal must touch nothing — calls %v poured %v", p.calls, rig.poured)
}
}
// The control: a restore that works pours nothing back, writes only the unit's definition, starts once, and leaves
// no copy behind.
func TestR1177_ASuccessfulRestoreIsUnchangedAndLeavesNoCopy(t *testing.T) {
m, p, rig, imported := r1177Fixture(t)
res, err := m.RestoreFromRecoveryUnit("app")
if err != nil {
t.Fatalf("restore: %v", err)
}
if len(rig.poured) != 0 || len(rig.removed) != 0 {
t.Fatalf("a successful restore poured back %v / removed %v", rig.poured, rig.removed)
}
if len(*imported) != 1 || res.VolumesReplayed != 2 {
t.Fatalf("the restore itself must be unchanged: imported %v volumes %d", *imported, res.VolumesReplayed)
}
if got := strings.Join(p.calls, ","); got != "stop,recreate,startsvc:immich-postgres,start" {
t.Fatalf("sequence = %q", got)
}
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
t.Fatalf("the copy must be removed after a successful restore, found %v", left)
}
}
// R-1195: a controller killed between the volume step and the database replay. What is on disk at the kill — the
// marker and the copy — is what the next start sees; Recover must pour the copy back BEFORE it starts the app.
// The kill is emulated by saving the marker and the copy at the replay (the moment of the cut), letting the run
// finish, then putting the saved disk back and starting a fresh guard over it.
// RED-PROOF: with Recover's put-back removed (or persistUndoForCrash not called) this fails on „nothing was poured
// back" — the pre-D22 restart left the app on the unit's data.
func TestR1195_AKilledLocalRestoreIsPutBackOnStart(t *testing.T) {
m, _, rig, _ := r1177Fixture(t)
marker := filepath.Join(t.TempDir(), "appstop-state.json")
m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0))
saved := map[string][]byte{}
m.importDBDump = func(context.Context, DiscoveredDB, string) error {
// the cut: the volumes are the unit's, the replay has begun
_ = filepath.Walk(filepath.Dir(rig.undoRoot), func(p string, fi os.FileInfo, _ error) error {
if fi != nil && !fi.IsDir() {
b, _ := os.ReadFile(p)
saved[p] = b
}
return nil
})
b, err := os.ReadFile(marker)
if err != nil {
t.Fatalf("no app-stop marker at the cut: %v", err)
}
saved[marker] = b
return nil
}
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
t.Fatalf("the run after the cut point: %v", err)
}
if len(saved) < 3 {
t.Fatalf("the copy was not on disk at the cut: %v", saved)
}
for p, b := range saved { // the disk as the kill left it
_ = os.MkdirAll(filepath.Dir(p), 0o700)
if err := os.WriteFile(p, b, 0o600); err != nil {
t.Fatal(err)
}
}
var calls []string
starter := starterFunc(func(name string) error { calls = append(calls, "start:"+name); return nil })
defs := []string{}
cp := NewCrashPutBack(func(name string) error { calls = append(calls, "stop:"+name); return nil },
func(name, dir string) error {
defs = append(defs, dir)
b, _ := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
calls = append(calls, "def:"+strings.TrimSpace(string(b)))
return nil
}, log.New(io.Discard, "", 0))
poured := map[string]string{}
cp.volReplay = func(_ string, dir string) (int, error) {
entries, _ := os.ReadDir(dir)
n := 0
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".tar") {
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
n++
}
}
return n, nil
}
var removed []string
cp.volRemove = func(v string) error { removed = append(removed, v); return nil }
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
g.SetStarter(starter)
g.SetPutBacker(cp)
rec := g.Recover()
if rec == nil || len(rec.PutBack) != 1 || rec.PutBack[0] != "app" {
t.Fatalf("the interrupted restore must be put back, got %+v", rec)
}
if poured[r1177Vol] != "LIVE bytes of "+r1177Vol {
t.Fatalf("nothing was poured back (or not the live bytes): %v", poured)
}
if len(removed) != 1 || removed[0] != r1177NewVol {
t.Fatalf("the unit's extra volume must be removed, removed %v", removed)
}
if got := strings.Join(calls, ","); got != "stop:app,def:LIVE compose,start:app" {
t.Fatalf("order = %q, want stop → live definition → start", got)
}
if left := undoCopies(t, rig.undoRoot); len(left) != 0 {
t.Fatalf("the used copy must be removed, found %v", left)
}
if _, err := os.Stat(marker); err == nil {
if b, _ := os.ReadFile(marker); strings.Contains(string(b), `"active": true`) {
t.Fatalf("the marker must be cleared after the put-back and start: %s", b)
}
}
}
// A cut BEFORE the copy was whole leaves no undo record: the old restart, nothing poured.
func TestR1195_ACutBeforeTheCopyRestartsAsBefore(t *testing.T) {
marker := filepath.Join(t.TempDir(), "appstop-state.json")
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil {
t.Fatal(err)
}
var calls []string
g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil }))
g2.SetPutBacker(NewCrashPutBack(func(string) error { calls = append(calls, "stop"); return nil }, nil, log.New(io.Discard, "", 0)))
rec := g2.Recover()
if rec == nil || len(rec.PutBack) != 0 || strings.Join(calls, ",") != "start:app" {
t.Fatalf("no copy → the plain restart, got %+v calls %v", rec, calls)
}
}
// A put-back that fails at start: the app is NOT started and the marker stays for the next start.
func TestR1195_AFailedPutBackOnStartLeavesTheAppStopped(t *testing.T) {
dir := t.TempDir()
marker := filepath.Join(dir, "appstop-state.json")
undo := filepath.Join(dir, "backups", "restore-undo", "app", "20261021T000000Z")
mustWrite(t, filepath.Join(undo, "undo.json"), `{"stack":"app","copied":["app_data"],"definition":false}`)
mustWrite(t, filepath.Join(undo, "app_data.tar"), "LIVE")
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil {
t.Fatal(err)
}
if err := g.SetUndo("app", undo); err != nil {
t.Fatal(err)
}
var calls []string
g2 := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
g2.SetStarter(starterFunc(func(n string) error { calls = append(calls, "start:"+n); return nil }))
cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0))
cp.volReplay = func(string, string) (int, error) { return 0, errors.New("disk gone") }
g2.SetPutBacker(cp)
rec := g2.Recover()
if rec == nil || len(rec.Failed) != 1 || len(calls) != 0 {
t.Fatalf("a failed put-back must not start the app: %+v calls %v", rec, calls)
}
if b, _ := os.ReadFile(marker); !strings.Contains(string(b), `"active": true`) {
t.Fatal("the marker must stay so the next start tries again")
}
if _, err := os.Stat(filepath.Join(undo, "app_data.tar")); err != nil {
t.Fatal("the copy must stay on disk after a failed put-back")
}
}
type starterFunc func(string) error
func (f starterFunc) StartStack(n string) error { return f(n) }
// R-1195's MEASURED case is the OFF-SITE full restore (rehearsal 2026-10-20 s2): the same cut, on that path — the
// marker and the copy on disk at the replay are what the next start pours back.
// RED-PROOF: without persistUndoForCrash in ReconstituteFromOffsite the marker holds no Undo → nothing poured back.
func TestR1195_AKilledOffsiteRestoreIsPutBackOnStart(t *testing.T) {
m, _, rig, _, _, _ := r893PutBackFixture(t)
marker := filepath.Join(t.TempDir(), "appstop-state.json")
m.appStop = NewAppStopGuard(marker, log.New(io.Discard, "", 0))
saved := map[string][]byte{}
m.importDBDump = func(context.Context, DiscoveredDB, string) error {
_ = filepath.Walk(rig.undoRoot, func(p string, fi os.FileInfo, _ error) error {
if fi != nil && !fi.IsDir() {
b, _ := os.ReadFile(p)
saved[p] = b
}
return nil
})
b, _ := os.ReadFile(marker)
saved[marker] = b
return nil
}
if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
t.Fatalf("the run after the cut point: %v", err)
}
for p, b := range saved {
_ = os.MkdirAll(filepath.Dir(p), 0o700)
_ = os.WriteFile(p, b, 0o600)
}
poured := map[string]string{}
cp := NewCrashPutBack(nil, nil, log.New(io.Discard, "", 0))
cp.volReplay = func(_ string, dir string) (int, error) {
entries, _ := os.ReadDir(dir)
n := 0
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".tar") {
b, _ := os.ReadFile(filepath.Join(dir, e.Name()))
poured[strings.TrimSuffix(e.Name(), ".tar")] = string(b)
n++
}
}
return n, nil
}
var removed []string
cp.volRemove = func(v string) error { removed = append(removed, v); return nil }
g := NewAppStopGuard(marker, log.New(io.Discard, "", 0))
g.SetStarter(starterFunc(func(string) error { return nil }))
g.SetPutBacker(cp)
rec := g.Recover()
if rec == nil || len(rec.PutBack) != 1 || poured[r893LiveVol] != "LIVE bytes of "+r893LiveVol {
t.Fatalf("the cut off-site restore must be put back on start: rec %+v poured %v", rec, poured)
}
if len(removed) != 1 || removed[0] != r893NewVol {
t.Fatalf("the snapshot's extra volume must be removed, removed %v", removed)
}
}
@@ -0,0 +1,55 @@
package backup
import (
"path/filepath"
"strings"
"testing"
"time"
)
// R-1178 (data-loss hunt 2026-10-20, R5): the second drive's „Teljes visszaállítás" merged files, THEN the unit half
// refused (a cut-off database copy), left the app STOPPED, and said „az alkalmazás és az adatai érintetlenek". Now the
// unit's refusals are asked before the file half: nothing moves, the app is never stopped, and the sentence is true.
// RED-PROOF: without the CheckOnly pre-check in RestoreTier2Whole this fails on „files changed before the refusal"
// (1 restored + 1 replaced, the measured R5 outcome — audits/dataloss-hunt-2026-10-20/C3-R5-whole-restore/).
func TestR1178_AUnitRefusalComesBeforeAnyFileMoves(t *testing.T) {
cut := strings.Replace(pgDump(1), "PostgreSQL database dump complete", "", 1) // a cut-off copy (R-640)
f := r102Tier2Fixture(t, []string{"app_data.tar"}, cut)
f.m.stackProvider = fileAppProvider{f.fake}
mid := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
mirrorLeg := filepath.Join(f.destBase, "hdd", "appdata", "app")
liveLeg := filepath.Join(f.liveDrive, "appdata", "app")
writeAt(t, filepath.Join(mirrorLeg, "p1.jpg"), "P1", mid)
writeAt(t, filepath.Join(mirrorLeg, "p2.jpg"), "P2 mirror NEWER", mid.Add(72*time.Hour))
writeAt(t, filepath.Join(liveLeg, "p2.jpg"), "P2 live older", mid.Add(48*time.Hour))
before := fingerprint(t, liveLeg)
res, err := f.m.RestoreTier2Whole("app")
if err == nil {
t.Fatal("a cut-off database copy must refuse the whole restore")
}
if res.Files.Restored+res.Files.Replaced != 0 {
t.Fatalf("files changed before the refusal: %+v", res.Files)
}
if after := fingerprint(t, liveLeg); !sameFingerprint(before, after) {
t.Fatalf("the live files changed: before %v after %v", before, after)
}
if f.fake.stopped {
t.Fatalf("the app was stopped by a restore that refused — calls %v", f.fake.calls)
}
if !strings.Contains(err.Error(), "rintetlenek") { // „érintetlenek" — now true
t.Fatalf("the refusal must keep its „untouched\" sentence (true now), got %v", err)
}
}
func sameFingerprint(a, b map[string]string) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if b[k] != v {
return false
}
}
return true
}
@@ -0,0 +1,60 @@
package backup
import (
"os"
"path/filepath"
"sort"
"testing"
)
// R-1180 (data-loss hunt 2026-10-20, T1): the nightly second-drive mirror (rsync -a --delete) ran DURING a restore
// from that same mirror and deleted the files the restore was still bringing back. Now RunAllTier2 skips an app
// while a restore of it runs (the restore record the pages follow), and mirrors it at the next run.
// RED-PROOF: without the restoreRunningFor skip the mirror becomes [a.jpg b.jpg] — c.jpg and d.jpg gone from the only
// copy (audits/dataloss-hunt-2026-10-20/C3-T1-mirror-during-restore/go-test-output.txt).
type r1180Provider struct{ *t2v2Provider }
func (p r1180Provider) ListDeployedStacks() []StackSummary { return []StackSummary{{Name: "app"}} }
func TestR1180_TheNightMirrorNeverRunsDuringARestoreOfThatApp(t *testing.T) {
m, src, target, prov := newTier2V2(t, "app")
prov.has["app"] = true
prov.binds["app"] = []ClassifiedBind{mHDD("appdata/app/photos")}
m.stackProvider = r1180Provider{prov}
m.tier2Mirror = rsyncMirror // the PRODUCTION mirror (rsync -a --delete), local temp dirs only
photos := filepath.Join(src, "appdata", "app", "photos")
for _, n := range []string{"a.jpg", "b.jpg", "c.jpg", "d.jpg"} {
mustWrite(t, filepath.Join(photos, n), n)
}
m.RunAllTier2()
mir := filepath.Join(target, "backups", "secondary", "app", "hdd", "appdata", "app", "photos")
ls := func(d string) []string {
e, _ := os.ReadDir(d)
var o []string
for _, x := range e {
o = append(o, x.Name())
}
sort.Strings(o)
return o
}
if got := ls(mir); len(got) != 4 {
t.Fatalf("positive control: the first night must mirror all four, got %v", got)
}
// The household lost two photos and started the second drive's restore of the app.
os.Remove(filepath.Join(photos, "c.jpg"))
os.Remove(filepath.Join(photos, "d.jpg"))
m.BeginRestoreOp("tier2-whole-restore", "app")
m.RunAllTier2() // the night fires while the restore is still copying
if got := ls(mir); len(got) != 4 {
t.Fatalf("the mirror ran during the restore and deleted what it was bringing back: %v", got)
}
// The restore ends; the next night mirrors again (the control that the skip is about the restore only).
m.EndRestoreOp(true, "")
m.RunAllTier2()
if got := ls(mir); len(got) != 2 {
t.Fatalf("after the restore the mirror must run again, got %v", got)
}
}
@@ -2,6 +2,7 @@ package backup
import (
"context"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"io"
"log"
"os"
@@ -321,7 +322,11 @@ func TestReconstituteDBOnlyStartFailureStillBringsTheStackUp(t *testing.T) {
// TestRestoreFromUnitReplayFailureStillBringsTheStackUp is the local path's version, and it also
// pins the pre-existing semantics: a replay error becomes a dataErr and surfaces as the "completed
// with data errors" outcome, with the app back up.
func TestRestoreFromUnitReplayFailureStillBringsTheStackUp(t *testing.T) {
// TestR1177_ReplayFailureWithNoCopyHoldsTheApp — D22 (2026-10-21) replaced this test's old outcome („completed with
// data errors" and the app started on the half-replayed database). A unit with no volume archive leaves nothing to
// copy, so a database replay that failed half-way cannot be put back: that is the R-379 half state, and the app is
// HELD stopped (operator ruling 2026-08-22), never started on it.
func TestR1177_ReplayFailureWithNoCopyHoldsTheApp(t *testing.T) {
m, prov, _ := r47UnitFixture(t, immichLikeCompose, true)
m.importDBDump = func(context.Context, DiscoveredDB, string) error {
return context.DeadlineExceeded
@@ -331,14 +336,14 @@ func TestRestoreFromUnitReplayFailureStillBringsTheStackUp(t *testing.T) {
if err == nil {
t.Fatal("a failed replay must be surfaced, not swallowed")
}
if !strings.Contains(err.Error(), "completed with data errors") {
t.Fatalf("the pre-existing outcome semantics must be preserved, got: %v", err)
if want := util.MsgError("err.backup.db_restore_failed_held_mixed", "app").Error(); err.Error() != want {
t.Fatalf("a half-replayed database with no copy must be HELD, got: %v", err)
}
if !prov.fullStarted {
t.Fatal("the stack was left DB-ONLY after a failed replay")
if prov.fullStarted {
t.Fatalf("the app was started on a half-replayed database — calls %v", prov.calls)
}
if got := strings.Join(prov.calls, ","); got != "stop,recreate,startsvc:immich-postgres,start" {
t.Fatalf("sequence = %q, want the full start to follow the failed replay", got)
if got := strings.Join(prov.calls, ","); got != "stop,recreate,startsvc:immich-postgres,stop" {
t.Fatalf("sequence = %q, want the DB-only window closed again (stop) and no full start", got)
}
}
@@ -3,6 +3,7 @@ package backup
import (
"context"
"errors"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"io"
"log"
"path/filepath"
@@ -194,12 +195,10 @@ func TestR638_FallbackVolumeFailureSkipsTheReplay(t *testing.T) {
// means the database's volume may still be the LIVE (possibly newer, migrated) one, or a half-filled
// one — the copy overlays it and leaves whatever it does not know about. So the importer must not run.
//
// What must NOT change from today's failure path: the error reaches the caller as "completed with data
// errors" (no silent success), the definition is the unit's, and the app is brought back up.
//
// COMPANION RED-PROOF: on the pre-fix restore_unit.go (dataErr set, then fall through to the replay)
// this fails on `the importer ran after a failed volume leg` — saved in
// audits/design-build-2026-10-06/B/red-slice2-no-replay-after-volume-failure.txt.
// Since D22 (2026-10-21, R-1177) the rest of the failure path changed on purpose: a volume that moved with no copy
// to pour back (this fixture's unit carries no archive, yet the replay reports one volume replaced) is the half
// state, so the app is HELD — no definition write, no start. With a copy the live volumes go back and the app
// starts (TestR1177_*).
func TestR638_UnitVolumeFailureNeverCallsTheImporter(t *testing.T) {
m, prov, imported := r47UnitFixture(t, immichLikeCompose, true)
m.volumeReplayFrom = func(string, string) (int, error) {
@@ -213,8 +212,8 @@ func TestR638_UnitVolumeFailureNeverCallsTheImporter(t *testing.T) {
if err == nil {
t.Fatal("a failed volume leg must be surfaced, not reported as success")
}
if !strings.Contains(err.Error(), "completed with data errors") || !strings.Contains(err.Error(), "app_immich_postgres_data") {
t.Fatalf("the pre-existing outcome must be preserved and name the failed volume, got: %v", err)
if want := util.MsgError("err.backup.restore_failed_held_mixed", "app").Error(); err.Error() != want {
t.Fatalf("a volume that moved with no copy must HOLD the app, got: %v", err)
}
if res.VolumesReplayed != 1 {
t.Errorf("the partial volume count must still be reported, got %d", res.VolumesReplayed)
@@ -222,14 +221,11 @@ func TestR638_UnitVolumeFailureNeverCallsTheImporter(t *testing.T) {
if res.DBsReplayed != 0 {
t.Errorf("no database was replayed, but the result says %d", res.DBsReplayed)
}
if prov.gotEnv == nil {
t.Error("the unit's definition must still be written, as on today's failure path")
if prov.fullStarted {
t.Fatal("the app was started on a half-restored volume")
}
if !prov.fullStarted {
t.Fatal("the app was left down — today's failure path brings it back up")
}
if got := strings.Join(prov.calls, ","); got != "stop,recreate,start" {
t.Fatalf("sequence = %q, want stop → recreate → full start (no DB-only window, no replay)", got)
if got := strings.Join(prov.calls, ","); got != "stop,stop" {
t.Fatalf("sequence = %q, want stop → (hold) stop: no definition write, no DB-only window, no start", got)
}
}
@@ -109,7 +109,8 @@ func undoCopies(t *testing.T, root string) []string {
t.Helper()
var out []string
_ = filepath.Walk(root, func(p string, fi os.FileInfo, _ error) error {
if fi != nil && !fi.IsDir() {
// the volume copies only — since R-1195 the copy also carries undo.json and the live definition
if fi != nil && !fi.IsDir() && strings.HasSuffix(p, ".tar") {
out = append(out, p)
}
return nil
@@ -0,0 +1,177 @@
package backup
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"time"
)
// D22 (operator ruling 2026-10-21 20:05, R-1195): a restore cut by a power loss puts the app back on start.
//
// MEASURED (rehearsal 2026-10-20, audits/rehearsal-2026-10-20/B6-failures/s2-power-cut-mid-restore/): a power cut 14 s
// into a full off-site restore left BookStack running on the backup's data — the household's newer page gone — while
// the page said only „start it again". The undo copy taken before the change sat on disk, unused, because nothing that
// runs at start knew it existed.
//
// Now every restore that takes the copy of the live volumes (offbox_restore_undo.go — the off-site restore and, since
// D22, the local unit restore) also, once the copy is whole:
// - copies the app's live definition files (docker-compose.yml, .felhom.yml, app.yaml — app.yaml keeps its secrets
// encrypted, as on disk) into <copy>/definition/;
// - writes <copy>/undo.json (which volumes were copied, which the backup adds);
// - records the copy's folder in the app-stop marker (AppStopGuard.SetUndo).
//
// At start, Recover finds the marker still active (the restore never reached its end) and calls
// PutBackInterruptedRestore BEFORE it restarts the app: stop, pour every copied volume back, remove the volumes the
// backup added, write the live definition back, remove the copy. A cut BEFORE the copy was whole leaves no Undo entry,
// and the old restart stays (nothing had moved yet). A put-back that fails is a failed recovery: the app is not
// started, the marker stays and the next start tries again.
//
// Pinned by r1195_crash_putback_test.go.
const (
undoJournalName = "undo.json"
undoDefinitionIn = "definition"
)
// undoJournal is what PutBackInterruptedRestore needs to know about a copy, written when the copy is whole.
type undoJournal struct {
Stack string `json:"stack"`
Copied []string `json:"copied"`
Absent []string `json:"absent,omitempty"`
Definition bool `json:"definition"`
At time.Time `json:"at"`
}
// undoDefinitionFiles are the files of an app's live definition the crash put-back writes back.
var undoDefinitionFiles = []string{"docker-compose.yml", ".felhom.yml", "app.yaml"}
// persistUndoForCrash makes the copy usable after a power cut (see the file comment). Best-effort: a failure is
// logged and the restore goes on — the in-process put-back does not need any of this.
func (m *Manager) persistUndoForCrash(stack string, c *liveVolumeCopy) {
if c == nil || c.dir == "" {
return
}
j := undoJournal{Stack: stack, Copied: c.copied, Absent: c.absent, At: time.Now().UTC()}
if m.stackProvider != nil {
if composePath, ok := m.stackProvider.GetStackComposePath(stack); ok && composePath != "" {
defDir := filepath.Join(c.dir, undoDefinitionIn)
if err := copyDefinitionFiles(filepath.Dir(composePath), defDir); err != nil {
m.logger.Printf("[WARN] [backup] %s: the live definition could not be kept beside the restore copy: %v — a power cut would put back the volumes only", stack, err)
} else {
j.Definition = true
}
}
}
data, err := json.MarshalIndent(j, "", " ")
if err == nil {
err = atomicWriteFile(filepath.Join(c.dir, undoJournalName), data)
}
if err != nil {
m.logger.Printf("[WARN] [backup] %s: the restore copy's journal could not be written: %v — a power cut would restart the app without putting it back", stack, err)
return
}
if err := m.appStop.SetUndo(stack, c.dir); err != nil {
m.logger.Printf("[WARN] [backup] %s: the restore copy could not be recorded for a power cut: %v", stack, err)
return
}
m.logger.Printf("[INFO] [backup] %s: the restore copy is recorded — a power cut from here on puts the app back on start (R-1195)", stack)
}
// copyDefinitionFiles copies the definition files that exist in stackDir into dst (0600). docker-compose.yml must exist.
func copyDefinitionFiles(stackDir, dst string) error {
if err := os.MkdirAll(dst, 0o700); err != nil {
return err
}
for _, name := range undoDefinitionFiles {
data, err := os.ReadFile(filepath.Join(stackDir, name))
if err != nil {
if os.IsNotExist(err) && name != "docker-compose.yml" {
continue
}
return err
}
if err := atomicWriteFile(filepath.Join(dst, name), data); err != nil {
return err
}
}
return nil
}
// atomicWriteFile writes data to path through a synced temp file and a rename (0600).
func atomicWriteFile(path string, data []byte) error {
tmp := path + ".tmp"
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
if err != nil {
return err
}
if _, err := f.Write(data); err != nil {
f.Close()
os.Remove(tmp)
return err
}
if err := f.Sync(); err != nil {
f.Close()
os.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
os.Remove(tmp)
return err
}
return os.Rename(tmp, path)
}
// CrashPutBack is the InterruptedRestorePutBacker the controller wires at start, before the backup manager exists.
type CrashPutBack struct {
stop func(stack string) error
writeDef func(stack, defDir string) error // writes the kept definition back (nil = volumes only)
logger *log.Logger
// seams for tests; nil = the restore's own volume helpers (Docker)
volReplay func(stack, dir string) (int, error)
volRemove func(vol string) error
}
// NewCrashPutBack builds the put-back. stop stops the app; writeDef writes a kept definition folder back as the app's.
func NewCrashPutBack(stop func(string) error, writeDef func(stack, defDir string) error, logger *log.Logger) *CrashPutBack {
return &CrashPutBack{stop: stop, writeDef: writeDef, logger: logger}
}
// PutBackInterruptedRestore pours undoDir back for stack and removes it. The app is left stopped.
func (p *CrashPutBack) PutBackInterruptedRestore(stack, undoDir string) error {
data, err := os.ReadFile(filepath.Join(undoDir, undoJournalName))
if err != nil {
return fmt.Errorf("reading the restore copy's journal: %w", err)
}
var j undoJournal
if err := json.Unmarshal(data, &j); err != nil {
return fmt.Errorf("reading the restore copy's journal: %w", err)
}
if j.Stack != stack {
return fmt.Errorf("the restore copy at %s belongs to %q, not %q", undoDir, j.Stack, stack)
}
if p.stop != nil {
if err := p.stop(stack); err != nil {
p.logger.Printf("[WARN] [appstop] %s: stopping before the put-back failed: %v (continuing)", stack, err)
}
}
mgr := &Manager{logger: p.logger, volumeReplayFrom: p.volReplay, undoVolumeRemoveFn: p.volRemove}
if err := mgr.putBack(stack, &liveVolumeCopy{dir: undoDir, copied: j.Copied, absent: j.Absent}); err != nil {
return err
}
if j.Definition && p.writeDef != nil {
if err := p.writeDef(stack, filepath.Join(undoDir, undoDefinitionIn)); err != nil {
return fmt.Errorf("writing the live definition back: %w", err)
}
}
p.logger.Printf("[WARN] [appstop] %s: the interrupted restore is put back from %s — %d volume(s) poured back, %d removed, definition=%v (R-1195)",
stack, undoDir, len(j.Copied), len(j.Absent), j.Definition)
if err := os.RemoveAll(undoDir); err != nil {
p.logger.Printf("[WARN] [appstop] %s: could not remove the used restore copy %s: %v", stack, undoDir, err)
} else {
_ = os.Remove(filepath.Dir(undoDir))
}
return nil
}
@@ -29,6 +29,10 @@ import (
// language at the moment it is written (release C, R-557).
const RestoreInterruptedKey = "note.restore.interrupted"
// RestoreInterruptedPutBackKey is what the household reads when the interrupted restore was put back at start
// (D22, R-1195): the data is as it was before the restore.
const RestoreInterruptedPutBackKey = "note.restore.interrupted_put_back"
// restoreRecordFile is the on-disk shape.
type restoreRecordFile struct {
Running bool `json:"running"`
@@ -49,6 +53,12 @@ func (m *Manager) SetRestoreRecordPath(path string) {
// LoadRestoreRecord restores the record at startup. It returns the restore that was interrupted by the
// stop — non-nil exactly once per interruption — so the caller can log it and raise restore_interrupted.
func (m *Manager) LoadRestoreRecord() *RestoreOpResult {
return m.LoadRestoreRecordAfterRecovery(nil)
}
// LoadRestoreRecordAfterRecovery is LoadRestoreRecord told which apps the start-up crash recovery put back
// (AppStopRecovery.PutBack, D22 / R-1195): for those the interrupted restore reads „the data is as before".
func (m *Manager) LoadRestoreRecordAfterRecovery(putBack []string) *RestoreOpResult {
m.mu.Lock()
defer m.mu.Unlock()
if m.opRecordPath == "" {
@@ -80,6 +90,11 @@ func (m *Manager) LoadRestoreRecord() *RestoreOpResult {
Op: rec.Op, Stack: rec.Stack, OK: false,
Message: m.note(RestoreInterruptedKey), FinishedAt: time.Now(), Interrupted: true,
}
for _, p := range putBack {
if p == rec.Stack {
res.Message, res.PutBack = m.note(RestoreInterruptedPutBackKey), true
}
}
m.opLast = &res
if m.opInterrupted == nil {
m.opInterrupted = map[string]RestoreOpResult{}
@@ -94,3 +94,26 @@ func TestRestoreRecord_NextRestoreOfThatAppClearsTheNotice(t *testing.T) {
t.Fatalf("starting gokapi's restore again did not clear its interrupted notice")
}
}
// D22 / R-1195: when the start-up crash recovery put the interrupted restore back, the record says the data is as it
// was before the restore — never „start it again" over data that is already back. Another app's interruption keeps
// the old sentence (the control).
// RED-PROOF: LoadRestoreRecordAfterRecovery ignoring putBack → the message keeps „indítsd el újra".
func TestR1195_TheRecordSaysTheDataIsAsBefore(t *testing.T) {
path := filepath.Join(t.TempDir(), "restore-status.json")
recordManager(path).BeginRestoreOp("offbox-restore", "bookstack")
got := recordManager(path).LoadRestoreRecordAfterRecovery([]string{"bookstack"})
if got == nil || !got.PutBack || !got.Interrupted {
t.Fatalf("want an interrupted, put-back record, got %+v", got)
}
if !strings.Contains(got.Message, "llapotban vannak") || strings.Contains(got.Message, "indítsd el újra") {
t.Fatalf("message %q must say the data is as before the restore", got.Message)
}
path2 := filepath.Join(t.TempDir(), "restore-status.json")
recordManager(path2).BeginRestoreOp("restore", "gokapi")
other := recordManager(path2).LoadRestoreRecordAfterRecovery([]string{"bookstack"})
if other == nil || other.PutBack || !strings.Contains(other.Message, "megszakadt") {
t.Fatalf("an app that was not put back keeps the old sentence, got %+v", other)
}
}
+113 -36
View File
@@ -149,6 +149,8 @@ func hasReplayableDump(dumpDir string) bool {
type UnitRestoreResult struct {
// VolumesReplayed is how many named-volume tars were unpacked into live Docker volumes.
VolumesReplayed int
// Stopped (R-1178) is set the moment the restore stops the app: false means it refused before touching anything.
Stopped bool
// DBsReplayed is how many .sql dumps were imported. Never inferred from the presence of a database
// service — only a completed import increments it.
DBsReplayed int
@@ -186,6 +188,10 @@ type UnitRestoreResult struct {
// branch with errors.Is. The customer sentence is the bundle's.
var ErrUnitVersionMismatch = errors.New("the recovery unit's definition is not the one its data belongs to")
// ErrRestoreBusy is the KIND of the refusal while another backup or restore holds the single-writer flag (W22: the
// household reads err.backup.restore_busy, never the English text).
var ErrRestoreBusy = errors.New("backup or restore already in progress")
// unitVersionCheck is A3's rule, as a pure function of the manifest and the unit's own compose file:
// known and matching → the pins to report; unknown → (nil, nil) and the caller WARNs; mixed or
// mismatched → the refusal.
@@ -278,6 +284,13 @@ type UnitRestoreOptions struct {
// It is a per-call argument and never a field on the Manager: a consent that outlives the act it
// was given for is not consent.
AcceptMissingFiles bool
// CheckOnly (R-1178) runs every refusal — the definition/data versions, the secrets, the database copy, the
// allow-list, the room for the live copy (D22) — and returns before anything is stopped or changed. The second
// drive's whole restore asks it BEFORE its file half, so a refusal never follows changed files.
CheckOnly bool
// FilesPlaced (R-1178) is how many files the caller already put back before this unit restore: a put-back then
// says those files stay, never „nothing changed".
FilesPlaced int
}
// ErrUnitLacksFileLegs is the refusal R-538 asks for. It names the app and the paths that are NOT in
@@ -321,7 +334,7 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
m.mu.Lock()
if m.running {
m.mu.Unlock()
return res, fmt.Errorf("backup or restore already in progress")
return res, util.MsgErrorf(ErrRestoreBusy, "err.backup.restore_busy", stackName)
}
m.running = true
m.mu.Unlock()
@@ -337,10 +350,14 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
// SOURCE path — that is what unitDir is for.
drivePath := m.GetAppDrivePath(stackName)
if drivePath == "" || !filepath.IsAbs(drivePath) {
return res, fmt.Errorf("cannot determine drive path for %s", stackName)
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s: cannot determine its drive path", stackName)
return res, util.MsgError("err.backup.restore_no_drive", stackName)
}
manifest := readManifest(UnitManifestFile(unitDir))
if manifest == nil && opt.CheckOnly {
return res, fmt.Errorf("no readable recovery unit for %s at %s", stackName, unitDir)
}
if manifest == nil {
m.logger.Printf("[WARN] [backup] No readable recovery unit for %s at %s — falling back to volume-only restore", stackName, unitDir)
m.mu.Lock()
@@ -422,13 +439,84 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
return res, util.MsgError("err.stacks.definition_refused", stackName, err.Error())
}
// Stop, restore named-volume data, recreate the definition, replay the DB with ONLY the database
// service running, and only then start the whole stack.
// F17: surface a data-restore failure instead of swallowing it (we still bring the app back up).
var dataErr error
// --- D22 (operator ruling 2026-10-21 20:05; R-1177): PUT BACK EXACTLY ON ANY FAILURE ------------------------
// The off-site restore's put-back (R-893 option A, offbox_restore_undo.go) now covers the local restore too.
// MEASURED before (audits/dataloss-hunt-2026-10-20/C3-R1-local-restore-9202/): a damaged volume tar → the live
// volume removed, `tar: short read`, the app started → Postgres crash-looped and the household's newest row was
// gone, with no copy anywhere. Now, BEFORE anything stops: the live volumes the unit will replace are measured and
// must fit beside the unit (the same rule as the off-site restore); the live definition is kept. RIGHT AFTER THE
// STOP they are copied; any failure after that pours them back, writes the live definition back and starts the
// app. A put-back that fails HOLDS the app and keeps the copy (unchanged R-893 hold). Pinned by
// r1177_local_putback_test.go.
nsRoot := m.namespaceRoot(drivePath)
undoVols := snapshotVolumeNames(UnitVolumeDumpDir(unitDir))
var undoExists map[string]bool
if len(undoVols) > 0 {
volNeed, ex, pErr := m.planLiveVolumeCopy(undoVols)
if pErr != nil {
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s: the live volumes could not be measured for their copy: %v — the app was NOT stopped", stackName, pErr)
return res, util.MsgError("err.backup.restore_undo_unmeasured", stackName)
}
if free := m.freeBytes(nsRoot); free >= 0 && free < volNeed {
m.logger.Printf("[WARN] [backup] Restore REFUSED for %s: the copy of the live volumes needs %d B, %d B is free on %s — the app was NOT stopped", stackName, volNeed, free, nsRoot)
return res, util.MsgError("err.backup.restore_undo_no_space", humanizeBytes(volNeed), humanizeBytes(free))
}
undoExists = ex
}
if opt.CheckOnly {
return res, nil
}
liveDef, ldErr := m.captureLiveDefinition(stackName)
if ldErr != nil {
// No live definition to keep (an app being brought back from a kept unit has none yet): a put-back then
// pours the volumes back under the unit's definition. Logged, never a refusal — the restore is its route back.
m.logger.Printf("[WARN] [backup] %s: no live definition to keep for a put-back (%v) — a failed restore would put back the volumes only", stackName, ldErr)
liveDef = nil
} else {
defer liveDef.cleanup()
}
// R-166: the stop→restore→start window is marked BEFORE the stop, so a controller killed inside it is owed a
// restart — and, once the copy is whole, a put-back (R-1195, restore_crash_putback.go).
if err := m.appStop.Begin("unit-restore:"+stackName, ReasonUnitRestore, []string{stackName}); err != nil {
return res, util.MsgError("err.backup.a_z_leallitasa_elotti_jelolo_nem", stackName, err)
}
restartStack := func() error {
err := m.stackProvider.StartStack(stackName)
if err == nil {
m.appStop.End()
} else {
m.appStop.ReleaseFailed(stackName)
}
return err
}
// Stop, copy the live volumes, restore named-volume data, recreate the definition, replay the DB with ONLY the
// database service running, and only then start the whole stack.
res.Stopped = true
if err := m.stackProvider.StopStack(stackName); err != nil {
m.logger.Printf("[WARN] [backup] could not stop %s before restore: %v (continuing)", stackName, err)
}
var undo *liveVolumeCopy
if len(undoVols) > 0 {
c, cErr := m.copyLiveVolumes(stackName, nsRoot, undoVols, undoExists)
if cErr != nil {
m.logger.Printf("[ERROR] [backup] %s: the copy of the live volumes failed: %v — nothing was restored, starting the app as it was", stackName, cErr)
if sErr := restartStack(); sErr != nil {
m.logger.Printf("[WARN] [backup] %s: restart after a failed volume copy also failed: %v", stackName, sErr)
}
return res, util.MsgError("err.backup.restore_undo_copy_failed", stackName)
}
undo = c
defer undo.drop(m, stackName)
m.persistUndoForCrash(stackName, undo) // R-1195: from here a power cut puts the app back on start
}
// moved: a live volume may have changed. With a copy that is always so (the replay removes before it recreates);
// without one (the unit holds no volume) only a database replay that began can have changed data — and with no
// copy to pour back that is the R-379 half state: held, never started (operator ruling 2026-08-22).
putBack := func(cause error, holdKey string, moved bool) (UnitRestoreResult, error) {
return res, m.putBackAfterFailedRestore(stackName, cause, liveDef, undo, moved || undo != nil, opt.FilesPlaced, restartStack, holdKey)
}
// R-353: the count is captured even when the replay errors — a partial replay is a fact the
// customer's sentence has to be built from, and discarding it on the error path is how Scenario C
// would end up wearing Scenario B's wording.
@@ -445,52 +533,41 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
replayed, volErr := volReplay(stackName, UnitVolumeDumpDir(unitDir))
res.VolumesReplayed = replayed
if volErr != nil {
m.logger.Printf("[ERROR] [backup] volume restore for %s: %v", stackName, volErr)
dataErr = volErr
// R-638 (option A, 09 §3 decision 154) still holds — the database copy is never replayed over a volume that is
// not known to be the copy's own — and since D22 nothing else runs on it either: the live volumes go back.
// Pinned by TestR638_UnitVolumeFailureNeverCallsTheImporter and TestR1177_*.
m.logger.Printf("[ERROR] [backup] volume restore for %s: %v — NOT replaying the database copy; putting the live data back", stackName, volErr)
return putBack(volErr, "err.backup.restore_failed_held_mixed", replayed > 0)
}
if err := m.stackProvider.RecreateStackDefinitionFromUnit(stackName, composeDir, fullEnv); err != nil {
return res, fmt.Errorf("recreating %s from unit: %w", stackName, err)
m.logger.Printf("[ERROR] [backup] recreating %s from unit: %v — putting the live data back", stackName, err)
return putBack(err, "err.backup.restore_failed_held_mixed", false)
}
// F17: the captured .sql dump is the authoritative logical DB state — replay it AFTER the volume
// restore, so the dump WINS over any volume-tar copy of the database.
// R-47: the replay happens with ONLY the database service up. This used to run after
// RecreateStackFromUnit had already brought the WHOLE stack up, letting the application rebuild
// schema objects underneath the replay (H4, DIAG-immich-restore-round2-2026-07-19).
//
// R-638 (option A, 09 §3 decision 154): NOT when the volume leg failed. The replay overlays the copy
// and removes only what the copy knows about, so it is safe only over the copy's OWN database files
// — which a failed volume leg does not guarantee (the volume may still be the live, newer one, or
// half-filled). The failure is already in dataErr and returned below; the definition is still the
// unit's and the app is still started, exactly as before. Pinned by
// TestR638_UnitVolumeFailureNeverCallsTheImporter.
if hasDumps && volErr != nil {
m.logger.Printf("[ERROR] [backup] %s: NOT replaying the database copy — the volume restore failed, so the database is not known to be the copy's own", stackName)
} else if hasDumps {
if hasDumps {
if err := m.stackProvider.StartStackServices(stackName, dbServices); err != nil {
m.logger.Printf("[ERROR] [backup] DB-only start for %s: %v", stackName, err)
if dataErr == nil {
dataErr = err
}
} else if n, err := m.reimportDBDumpsAtCtx(stackName, dbDumpDir); err != nil {
res.DBsReplayed = n // partial credit: whatever imported before the failure really did import
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v", stackName, err)
if dataErr == nil {
dataErr = err
}
} else {
res.DBsReplayed = n
m.logger.Printf("[ERROR] [backup] DB-only start for %s: %v — putting the live data back", stackName, err)
return putBack(err, "err.backup.restore_failed_held_mixed", false)
}
n, err := m.reimportDBDumpsAtCtx(stackName, dbDumpDir)
res.DBsReplayed = n // partial credit: whatever imported before the failure really did import
if err != nil {
m.logger.Printf("[ERROR] [backup] DB re-import for %s: %v — putting the live data back", stackName, err)
return putBack(err, "err.backup.db_restore_failed_held_mixed", true)
}
}
if err := m.stackProvider.StartStack(stackName); err != nil {
return res, fmt.Errorf("starting %s after restore from unit: %w", stackName, err)
if err := restartStack(); err != nil {
m.logger.Printf("[ERROR] [backup] starting %s after restore from unit: %v — putting the live data back", stackName, err)
return putBack(err, "err.backup.restore_failed_held_mixed", false)
}
if err := m.waitForHealthy(stackName, healthTimeout); err != nil {
m.logger.Printf("[WARN] [backup] %s restored but health check failed: %v", stackName, err)
}
if dataErr != nil {
return res, fmt.Errorf("restore of %s from unit completed with data errors: %w", stackName, dataErr)
}
m.logger.Printf("[INFO] [backup] Restore-from-unit completed: %s — %d volume(s) of %d listed, %d database(s) of %d listed",
stackName, res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
// Slice 4: the app is back on its unit's definition and data and was started — the route back an
+24
View File
@@ -528,6 +528,12 @@ func (m *Manager) RunAllTier2() {
m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — the app is HELD; its copy is the restore point and is preserved", stack.Name)
continue
}
// R-1180: never mirror an app while a restore of it runs — the mirror (rsync --delete) would remove files the
// restore is still bringing back from that same copy. Skipped and said; the next night mirrors it.
if m.restoreRunningFor(stack.Name) {
m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — a restore of it is running; it is mirrored at the next run (R-1180)", stack.Name)
continue
}
// R-645: the mirror of an app not running its pinned version stays as it is.
if why, skip := m.versionSkip(stack.Name); skip {
m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — it is not running its pinned version (%s); the last good copy is kept (R-645)", stack.Name, why)
@@ -546,6 +552,10 @@ func (m *Manager) RunAllTier2() {
// R-7b: the SHARES source runs after the per-app loop, in the SAME orchestrator run. It is a
// sibling job — nothing above it changed — and its failure never fails the app tier.
if m.restoreRunningFor(SharesPseudoStack) {
m.logger.Printf("[WARN] [backup] Tier 2 shares job skipped — a restore of the shares is running; mirrored at the next run (R-1180)")
return
}
if err := m.RunSharesTier2(); err != nil {
m.logger.Printf("[WARN] [backup] Tier 2 shares job failed: %v", err)
}
@@ -811,3 +821,17 @@ func dirSizeBytes(dir string) int64 {
}
return size
}
// restoreRunningFor reports whether a restore of stack is in flight (the restore record the pages follow, R-550).
// The shares restores are recorded under their display name.
func (m *Manager) restoreRunningFor(stack string) bool {
m.mu.Lock()
defer m.mu.Unlock()
if !m.opRunning {
return false
}
if stack == SharesPseudoStack {
return m.opStack == SharesDisplayName || m.opStack == SharesPseudoStack
}
return m.opStack == stack
}
+20 -3
View File
@@ -304,6 +304,13 @@ func (m *Manager) RestoreTier2Whole(stackName string) (WholeRestoreResult, error
m.logger.Printf("[WARN] [backup] whole restore REFUSED for %s: the file half needs %d B and %d B is free (floor %d B) — nothing moved", stackName, need, free, wholeRestoreFloorBytes)
return res, &wholeRestoreSpace{need: need, free: free}
}
// R-1178: the unit half's refusals are asked HERE, before the file half — measured before: files merged, then the
// unit refused (a cut-off database copy) and the app was left STOPPED under a page that said „untouched".
unitDir := tier2UnitDir(destBase)
if _, err := m.RestoreFromRecoveryUnitAtWith(stackName, unitDir, UnitRestoreOptions{AcceptMissingFiles: true, CheckOnly: true}); err != nil {
m.logger.Printf("[WARN] [backup] whole restore REFUSED for %s before the file half: the unit half would refuse (%v) — nothing moved", stackName, err)
return res, err
}
// THE FILE HALF, under the backup single-flight, with the app stopped.
if err := m.acquireRunning(); err != nil {
@@ -341,9 +348,19 @@ func (m *Manager) RestoreTier2Whole(stackName string) (WholeRestoreResult, error
return res, util.MsgError("err.backup.fajlmasolas_sikertelen", mergeErr)
}
// THE UNIT HALF — the existing restore, told that the files are handled (R-538 stays for every other caller).
unitDir := tier2UnitDir(destBase)
res.Unit, err = m.RestoreFromRecoveryUnitAtWith(stackName, unitDir, UnitRestoreOptions{AcceptMissingFiles: true})
// THE UNIT HALF — the existing restore, told that the files are handled (R-538 stays for every other caller) and
// how many it placed (a put-back then says those files stay, D22).
placed := res.Files.Restored + res.Files.Replaced
res.Unit, err = m.RestoreFromRecoveryUnitAtWith(stackName, unitDir, UnitRestoreOptions{AcceptMissingFiles: true, FilesPlaced: placed})
if err != nil && !res.Unit.Stopped {
// R-1178: the unit refused before touching anything (asked above, so only a change in between gets here) —
// but the file half HAS changed files and stopped the app. Start it, and say what changed.
m.logger.Printf("[ERROR] [backup] whole restore %s: the unit half refused after the file half (%v) — starting the app; %d file(s) restored, %d replaced", stackName, err, res.Files.Restored, res.Files.Replaced)
if startErr := m.stackProvider.StartStack(stackName); startErr != nil {
m.logger.Printf("[ERROR] [backup] restarting %s after the unit half refused also failed: %v", stackName, startErr)
}
err = util.MsgError("err.backup.whole_restore_unit_refused_files_changed", stackName, res.Files.Restored, res.Files.Replaced)
}
m.rehydratePrimaryUnit(stackName, unitDir, err)
return res, err
}