R-1251 (P1): a pending put-back blocks every start — a cut restore's copy on an unbound drive waits (a restore hold), is poured back when the drive binds before any start; a failed put-back holds the app (H2); Stop during a restore still puts back (H7). Commit only, unreleased.
gates / gates (push) Successful in 1m18s

This commit is contained in:
2026-10-24 19:07:02 +02:00
parent 2449223f10
commit bb2809dc68
16 changed files with 847 additions and 24 deletions
+70 -20
View File
@@ -133,6 +133,10 @@ type AppStopGuard struct {
// putBacker (D22, R-1195) pours a restore's undo copy back before Recover restarts the app. nil = no put-back
// (the pre-D22 restart, unchanged).
putBacker InterruptedRestorePutBacker
// holds and undoReachable (R-1251, appstop_putback_wait.go): where a waiting or failed put-back is HELD, and
// whether the drive a copy lives on is bound. nil = the pre-R-1251 behaviour.
holds AppStopHolds
undoReachable func(stack, dir string) bool
// suppressed (R-330) is the in-memory "the alarm must not fire for these, we are holding them"
// set. It is deliberately NOT persisted: after a restart the guard no longer holds anything —
@@ -162,6 +166,11 @@ type AppStopRecovery struct {
// PutBack are apps whose interrupted restore was undone from its copy before the restart (D22, R-1195): their
// data is as it was before the restore.
PutBack []string
// Waiting (R-1251) are apps whose restore copy is on a drive that is not bound: HELD, not started, put back first
// when the drive binds (ResumePutBack). Like Refused, the holder owns them and they do not alarm.
Waiting []string
// Held (R-1251 H2) are apps whose put-back FAILED and that are now held for support (the D20 hold). They alarm.
Held []string
}
// Alarming reports whether this recovery is worth paging an operator about. A recovery that only
@@ -171,7 +180,7 @@ func (r *AppStopRecovery) Alarming() bool {
if r == nil {
return false
}
return len(r.Failed) > 0 || len(r.Restarted) > 0
return len(r.Failed) > 0 || len(r.Held) > 0 || len(r.Restarted) > 0
}
// Message is the operator-facing headline for an interrupted operation.
@@ -179,14 +188,22 @@ func (r *AppStopRecovery) Message() string {
if r == nil {
return ""
}
if len(r.Failed) > 0 {
if len(r.Failed) > 0 || len(r.Held) > 0 {
bad := len(r.Failed) + len(r.Held)
m := fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted",
r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed))
r.Reason.humanReason(), bad, len(r.Restarted)+bad)
if len(r.Held) > 0 {
m += fmt.Sprintf(" (%d could not be put back and are HELD for support)", len(r.Held))
}
if len(r.Refused) > 0 {
m += fmt.Sprintf(" (a further %d are held by an absent drive and are not counted as failures)", len(r.Refused))
}
return m
}
if len(r.Waiting) > 0 && len(r.Restarted) == 0 {
return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are HELD until their data drive is bound: the copy taken before the change is on it and is put back first",
r.Reason.humanReason(), len(r.Waiting))
}
if len(r.Refused) > 0 && len(r.Restarted) == 0 {
return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are left stopped and HELD: their data drive is not available, so the drive gate restarts them when it returns",
r.Reason.humanReason(), len(r.Refused))
@@ -215,6 +232,12 @@ func (r *AppStopRecovery) Detail() string {
if len(r.PutBack) > 0 {
d += fmt.Sprintf(" put_back=%v", r.PutBack)
}
if len(r.Waiting) > 0 {
d += fmt.Sprintf(" put_back_waiting_for_drive=%v", r.Waiting)
}
if len(r.Held) > 0 {
d += fmt.Sprintf(" put_back_failed_held=%v", r.Held)
}
return d
}
@@ -352,28 +375,48 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt}
hs, _ := g.starter.(interface{ WantsStopped(string) bool })
var usedUndo []string // apps whose copy was poured back: the marker must not name it again
for _, name := range m.Stacks {
if hs != nil && hs.WantsStopped(name) {
// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down.
// It is owed nothing; the marker still clears below.
g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
continue
}
// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down — it is owed
// no restart. R-1251 H7: it is still owed its put-back, so the stop is honoured AFTER the copy is poured back.
wantsStopped := hs != nil && hs.WantsStopped(name)
if dir := m.Undo[name]; dir != "" && g.putBacker != nil {
// D22 (R-1195): the restore was cut after its copy of the live data was taken — the app may be on the
// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery
// (the marker is kept, so the next startup tries again) and the app is not started on unknown data.
if _, statErr := os.Stat(dir); statErr != nil {
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
} else if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
res.Failed = append(res.Failed, name)
// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery and the
// app is not started on unknown data.
state, statErr := g.copyState(name, dir)
switch state {
case undoWaiting:
// R-1251: the copy is on a drive that is not bound yet. It is not „gone": the app is HELD (every start
// path honours the hold) and the copy is poured back when the drive binds (ResumePutBack).
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s cannot be reached yet (%v) — NOT starting it; the put-back waits for its drive", name, dir, statErr)
g.holdPutBackPending(name, dir)
res.Waiting = append(res.Waiting, name)
continue
} else {
case undoGone:
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
default:
if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
// R-1251 H2: HELD (the D20 hold), never stopped-and-unheld. Without a hold store the old way stays:
// Failed, and the marker is kept so the next start tries again.
if g.holdPutBackFailed(name, dir, err) {
res.Held = append(res.Held, name)
} else {
res.Failed = append(res.Failed, name)
}
continue
}
g.logger.Printf("[WARN] [appstop] crash recovery: %s's interrupted restore is undone — its data is as before the restore (R-1195)", name)
g.clearPutBackPending(name)
res.PutBack = append(res.PutBack, name)
usedUndo = append(usedUndo, name)
}
}
if wantsStopped {
g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
continue
}
if err := g.starter.StartStack(name); err != nil {
// R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be
// started (an absent data drive), so the app is left down deliberately and the holder
@@ -395,6 +438,8 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
sort.Strings(res.Refused)
sort.Strings(res.Restarted)
sort.Strings(res.PutBack)
sort.Strings(res.Waiting)
sort.Strings(res.Held)
// The marker is kept for BOTH unfinished outcomes, for the same reason and with different
// urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished
@@ -403,11 +448,16 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
if len(res.Failed) > 0 {
g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v",
len(res.Failed), res.Failed)
g.dropFromMarker(res.Held...)
g.forgetUndo(usedUndo...)
return res
}
if len(res.Refused) > 0 {
g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent) — KEEPING the marker; this is the gate working, not a fault: %v",
len(res.Refused), res.Refused)
if len(res.Refused) > 0 || len(res.Waiting) > 0 {
g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent: %v; put-back waiting for its drive: %v) — KEEPING the marker; this is the gate working, not a fault",
len(res.Refused)+len(res.Waiting), res.Refused, res.Waiting)
// The held (H2) apps and the used copies leave the marker; the rest of it stays for the next start.
g.dropFromMarker(res.Held...)
g.forgetUndo(usedUndo...)
return res
}
g.End()
@@ -0,0 +1,285 @@
package backup
import (
"fmt"
"os"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// ── R-1251: a pending put-back blocks every start (review round 2, HUNT-1 H1) ─────────────────────────────────────
//
// D22 pours a cut restore's copy back at start (restore_crash_putback.go). For a DRIVE-backed app the copy lives on
// the app's drive (restoreUndoDir under the drive's namespace), and at start the drive is often not bound yet. Until
// this file Recover read the missing folder as „gone", went on to the gated start (refused: no drive) and kept the
// marker; the drive then bound, the boot recreate started the app on the cut restore's data (it asked only „held?"),
// the household wrote, and the NEXT controller start found the copy and poured it over everything written since.
//
// Now:
// - a copy that cannot be seen while its drive is not bound is WAITING, never gone. Recover writes a restore hold
// (settings.HoldReasonPutBackPending, the copy's folder in SafetyDump) and starts nothing. The hold is the ONE
// storage every start path already honours — the household's Start/Restart/Update, the boot recreate, the drive
// return, the media restart, the boot sweep, the night chain, the update leg — so no path needs its own check;
// - when the drive binds, ResumePutBack pours the copy back FIRST and lifts the hold; only then may the app start;
// - a put-back that FAILS (at start or on resume) leaves the app HELD with the D20 hold (HoldReasonRestoreMixed, the
// household's „stopped half-way" sentence) and the copy on disk — never stopped-and-unheld, which Start or the boot
// recreate would run and a later start would pour an old copy over (H2). The marker then lets go of that app: the
// hold owns it, exactly as holdAppAfterMixedRestore does in-process;
// - Stop pressed during the restore no longer skips the put-back after a cut (H7): the data is put back, the app
// stays stopped as the household asked.
//
// The hold — not the marker — is the durable record of a waiting put-back, because the marker has one slot and the
// next operation's Begin replaces it (R-1253).
//
// Pinned by r1251_putback_waits_test.go.
// AppStopHolds is the hold store the guard writes (settings.Settings in production).
type AppStopHolds interface {
SetRestoreHold(settings.RestoreHold) error
GetRestoreHold(stack string) (settings.RestoreHold, bool)
ClearRestoreHold(stack string) (bool, error)
ListRestoreHolds() []settings.RestoreHold
}
// SetHolds wires the hold store. INIT-ONLY, before Recover. nil = the pre-R-1251 behaviour (no hold is written).
func (g *AppStopGuard) SetHolds(h AppStopHolds) {
if g != nil {
g.holds = h
}
}
// SetUndoReachable wires the question „is the drive this app's restore copy lives on bound right now?" (main.go:
// the app's HDD_PATH is a live mountpoint). nil = every folder counts as reachable, so a missing copy is „gone".
func (g *AppStopGuard) SetUndoReachable(f func(stack, dir string) bool) {
if g != nil {
g.undoReachable = f
}
}
type undoCopyState int
const (
undoPresent undoCopyState = iota
undoGone // its drive is bound and the folder is not there
undoWaiting // its drive is not bound (or the folder cannot be read): it may come back
)
func (s undoCopyState) String() string {
switch s {
case undoPresent:
return "present"
case undoGone:
return "gone"
default:
return "waiting"
}
}
// copyState decides present / gone / waiting for one recorded copy.
func (g *AppStopGuard) copyState(stack, dir string) (undoCopyState, error) {
_, err := os.Stat(dir)
if err == nil {
return undoPresent, nil
}
if !os.IsNotExist(err) {
return undoWaiting, err // an unreadable folder is not proof that it is gone
}
if g.undoReachable != nil && !g.undoReachable(stack, dir) {
return undoWaiting, err
}
return undoGone, err
}
// holdPutBackPending writes the „waiting for the drive" hold. Returns whether the app is now held by SOME hold.
func (g *AppStopGuard) holdPutBackPending(stack, dir string) bool {
if g.holds == nil {
g.logger.Printf("[ERROR] [appstop] %s: its restore copy %s waits for its drive, but no hold store is wired — nothing durable refuses a start (R-1251)", stack, dir)
return false
}
if h, ok := g.holds.GetRestoreHold(stack); ok && h.Reason != settings.HoldReasonPutBackPending {
// Another hold already refuses every start, and its sentence names the harder problem. The marker keeps the
// copy's folder, so the next start puts it back once the drive is there.
g.logger.Printf("[WARN] [appstop] %s: its restore copy %s waits for its drive; the app is already held (%q, set %s) — that hold stays", stack, dir, h.Reason, h.At)
return true
}
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending, SafetyDump: dir}
if err := g.holds.SetRestoreHold(h); err != nil {
g.logger.Printf("[ERROR] [appstop] %s: the waiting put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
return false
}
g.logger.Printf("[WARN] [appstop] %s: the cut restore's copy %s is on a drive that is not bound — the app is HELD until the copy is put back (R-1251)", stack, dir)
return true
}
// holdPutBackFailed writes the D20 hold for a put-back that could not be done (H2). Returns whether it was written.
func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
if g.holds == nil {
return false
}
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
if err := g.holds.SetRestoreHold(h); err != nil {
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
return false
}
g.logger.Printf("[ERROR] [appstop] %s: the interrupted restore could NOT be put back (%v) — the app is HELD for support, the copy stays at %s (R-1251 H2)", stack, cause, dir)
return true
}
// clearPutBackPending lifts this app's waiting-put-back hold, and only that kind.
func (g *AppStopGuard) clearPutBackPending(stack string) {
if g.holds == nil {
return
}
if h, ok := g.holds.GetRestoreHold(stack); !ok || h.Reason != settings.HoldReasonPutBackPending {
return
}
if _, err := g.holds.ClearRestoreHold(stack); err != nil {
g.logger.Printf("[ERROR] [appstop] %s: the put-back ran, but its hold could not be cleared: %v — the app stays held", stack, err)
}
}
// dropFromMarker lets the active marker forget these apps (and their copies): a hold now owns them, or their put-back
// is done. A marker left with no app is cleared. A marker that belongs to another operation (it does not hold them)
// is not touched.
func (g *AppStopGuard) dropFromMarker(names ...string) {
if g == nil || g.path == "" || len(names) == 0 {
return
}
m, ok := g.read()
if !ok || !m.Active {
return
}
drop := map[string]bool{}
for _, n := range names {
drop[n] = true
}
var keep []string
for _, s := range m.Stacks {
if !drop[s] {
keep = append(keep, s)
}
}
if len(keep) == len(m.Stacks) {
return
}
if len(keep) == 0 {
if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) {
g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v", g.path, err)
}
return
}
m.Stacks = keep
for n := range drop {
delete(m.Undo, n)
}
if err := g.write(m); err != nil {
g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without %v: %v", names, err)
}
}
// forgetUndo removes these apps' copy folders from the active marker (their put-back is done; the copy is removed),
// so a later start never tries to pour back a folder that no longer holds the app's data.
func (g *AppStopGuard) forgetUndo(names ...string) {
if g == nil || g.path == "" || len(names) == 0 {
return
}
m, ok := g.read()
if !ok || !m.Active || len(m.Undo) == 0 {
return
}
changed := false
for _, n := range names {
if _, ok := m.Undo[n]; ok {
delete(m.Undo, n)
changed = true
}
}
if !changed {
return
}
if err := g.write(m); err != nil {
g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without the used copies of %v: %v", names, err)
}
}
// resumeMu serialises ResumePutBack: the boot recreate and the drive-return sweep can reach the same app at once, and
// a copy must be poured back exactly once.
var resumeMu sync.Mutex
// ResumePutBack runs a put-back that was WAITING for its drive. It returns true when nothing is owed for stack any
// more — there was no waiting put-back, or it ran now — and the caller may go on to start the app (its other holds
// still apply). false: the app stays held (the drive is still not bound, or the put-back failed and the app is now
// held for support). Nil-safe.
func (g *AppStopGuard) ResumePutBack(stack string) bool {
if g == nil || g.holds == nil {
return true
}
resumeMu.Lock()
defer resumeMu.Unlock()
h, ok := g.holds.GetRestoreHold(stack)
if !ok || h.Reason != settings.HoldReasonPutBackPending {
return true
}
dir := h.SafetyDump
if m, ok := g.read(); ok && m.Active && m.Undo[stack] != "" {
dir = m.Undo[stack]
}
if dir == "" || g.putBacker == nil {
g.logger.Printf("[ERROR] [appstop] %s: a put-back waits, but its copy is not named (or no put-back is wired) — the app stays HELD", stack)
return false
}
state, serr := g.copyState(stack, dir)
switch state {
case undoWaiting:
g.logger.Printf("[INFO] [appstop] %s: the restore copy %s is still not reachable (%v) — the app stays held until its drive is bound", stack, dir, serr)
return false
case undoGone:
g.holdPutBackFailed(stack, dir, fmt.Errorf("the restore copy %s is gone while its drive is bound", dir))
g.dropFromMarker(stack)
return false
}
if err := g.putBacker.PutBackInterruptedRestore(stack, dir); err != nil {
if g.holdPutBackFailed(stack, dir, err) {
g.dropFromMarker(stack)
}
return false
}
g.clearPutBackPending(stack)
g.dropFromMarker(stack)
g.logger.Printf("[WARN] [appstop] %s: the drive is bound — the interrupted restore is put back from %s BEFORE the app starts; its data is as before the restore (R-1251)", stack, dir)
return true
}
// ResumeWaitingPutBacks runs every waiting put-back whose drive is bound now, and starts each app afterwards through
// the guard's gated starter — unless the household stopped it. Called at start (after Recover) and when a drive
// returns. Returns the apps started.
func (g *AppStopGuard) ResumeWaitingPutBacks() []string {
if g == nil || g.holds == nil {
return nil
}
var started []string
for _, h := range g.holds.ListRestoreHolds() {
if h.Reason != settings.HoldReasonPutBackPending {
continue
}
if !g.ResumePutBack(h.Stack) {
continue
}
if ws, ok := g.starter.(interface{ WantsStopped(string) bool }); ok && ws.WantsStopped(h.Stack) {
g.logger.Printf("[INFO] [appstop] %s: put back; NOT starting it — the household stopped it", h.Stack)
continue
}
if g.starter == nil {
continue
}
if err := g.starter.StartStack(h.Stack); err != nil {
g.logger.Printf("[WARN] [appstop] %s: put back, but the start was not done: %v", h.Stack, err)
continue
}
started = append(started, h.Stack)
}
return started
}
@@ -369,6 +369,9 @@ func (m *Manager) RestoreHoldForLang(stack, lang string) (bool, string) {
if t, err := time.Parse(time.RFC3339, h.At); err == nil {
when = t.Format("2006-01-02 15:04")
}
if h.Reason == settings.HoldReasonPutBackPending { // R-1251: the put-back waits for the copy's drive
return true, util.Text(lang, "hold.putback_pending", stack)
}
if h.Reason == settings.HoldReasonRestoreMixed { // R-893
return true, util.Text(lang, "note.reconstitute.held_mixed", stack, when)
}
@@ -0,0 +1,320 @@
package backup
// R-1251 (review round 2, HUNT-1 H1, H2, H7): a pending put-back blocks every start. The first test is the review's
// lean-clone proof (audits/review-round-2-2026-10-24/proofs/H1-drive-cut-restore/) turned round: the same cut, the same
// unbound drive, and now the app is HELD until the copy is poured back, and a later start pours nothing.
//
// RED-PROOF (2026-10-24, each seen failing on main 2449223 before appstop_putback_wait.go):
// - TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart: „boot 1 must HOLD the app as waiting" —
// Recover read the copy as gone and only REFUSED the start, with no hold;
// - TestR1251_AFailedPutBackHoldsTheApp: „a failed put-back must leave a restore_mixed hold" — none was written;
// - TestR1251_StopPressedDuringTheRestoreStillPutsBack: „the put-back must run" — R-721's skip came first.
import (
"errors"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// r1251Rig is one app (immich) whose restore was cut after its copy was recorded, the copy on the app's drive.
type r1251Rig struct {
marker, drive, undo string
sett *settings.Settings
puts []string
starts []string
stopped map[string]bool
putErr error
}
func newR1251Rig(t *testing.T) *r1251Rig {
t.Helper()
root := t.TempDir()
r := &r1251Rig{
marker: filepath.Join(root, "appstop-state.json"),
drive: filepath.Join(root, "drive"),
stopped: map[string]bool{},
}
r.undo = filepath.Join(r.drive, "felhom-data", "backups", "restore-undo", "immich", "20261024T120000Z")
mustWrite(t, filepath.Join(r.undo, "undo.json"), `{"stack":"immich","copied":["immich_pgdata"]}`)
sett, err := settings.Load(filepath.Join(root, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
r.sett = sett
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
if err := g.Begin("unit-restore:immich", ReasonUnitRestore, []string{"immich"}); err != nil {
t.Fatal(err)
}
if err := g.SetUndo("immich", r.undo); err != nil {
t.Fatal(err)
}
return r // the power cut: no End
}
func (r *r1251Rig) unbind(t *testing.T) {
t.Helper()
if err := os.Rename(r.drive, r.drive+".unbound"); err != nil {
t.Fatal(err)
}
}
func (r *r1251Rig) bind(t *testing.T) {
t.Helper()
if err := os.Rename(r.drive+".unbound", r.drive); err != nil {
t.Fatal(err)
}
}
func (r *r1251Rig) driveBound() bool { _, err := os.Stat(r.drive); return err == nil }
// r1251Starter is the gated starter: it refuses while the drive is absent or a hold is set (as driveStartGate does).
type r1251Starter struct{ r *r1251Rig }
func (s r1251Starter) StartStack(name string) error {
if _, held := s.r.sett.GetRestoreHold(name); held {
return ErrStartRefused
}
if !s.r.driveBound() {
return ErrStartRefused
}
s.r.starts = append(s.r.starts, name)
return nil
}
func (s r1251Starter) WantsStopped(name string) bool { return s.r.stopped[name] }
type r1251PutBacker struct{ r *r1251Rig }
func (p r1251PutBacker) PutBackInterruptedRestore(stack, dir string) error {
if p.r.putErr != nil {
return p.r.putErr
}
p.r.puts = append(p.r.puts, stack)
return os.RemoveAll(dir) // the real put-back removes the used copy
}
// guard is a fresh controller start over the rig's disk, wired as main.go wires it.
func (r *r1251Rig) guard() *AppStopGuard {
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
g.SetStarter(r1251Starter{r})
g.SetPutBacker(r1251PutBacker{r})
g.SetHolds(r.sett)
g.SetUndoReachable(func(string, string) bool { return r.driveBound() })
return g
}
func (r *r1251Rig) markerActive() bool {
b, err := os.ReadFile(r.marker)
return err == nil && strings.Contains(string(b), `"active": true`)
}
func TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart(t *testing.T) {
r := newR1251Rig(t)
// boot 1: the drive is not bound yet
r.unbind(t)
g1 := r.guard()
rec := g1.Recover()
h, held := r.sett.GetRestoreHold("immich")
if rec == nil || len(rec.Waiting) != 1 || !held || h.Reason != settings.HoldReasonPutBackPending {
t.Fatalf("boot 1 must HOLD the app as waiting for its drive: rec=%+v hold=%+v held=%v", rec, h, held)
}
if h.SafetyDump != r.undo {
t.Fatalf("the hold must name the copy, got %q", h.SafetyDump)
}
if len(r.starts) != 0 || len(r.puts) != 0 || rec.Alarming() {
t.Fatalf("nothing may start or be poured while the drive is absent: starts=%v puts=%v alarming=%v", r.starts, r.puts, rec.Alarming())
}
if !r.markerActive() {
t.Fatal("the marker must be kept while the put-back waits")
}
// a resume while the drive is still absent does nothing and keeps the hold
if g1.ResumePutBack("immich") {
t.Fatal("ResumePutBack must say still-held while the copy cannot be reached")
}
if _, still := r.sett.GetRestoreHold("immich"); !still {
t.Fatal("the hold must stay while the drive is absent")
}
// the drive binds: the boot recreate (and the drive-return sweep) call ResumePutBack before they start the app
r.bind(t)
if !g1.ResumePutBack("immich") {
t.Fatal("with the drive bound the put-back must run and allow the start")
}
if strings.Join(r.puts, ",") != "immich" {
t.Fatalf("the copy must be poured back exactly once, puts=%v", r.puts)
}
if _, still := r.sett.GetRestoreHold("immich"); still {
t.Fatal("the waiting hold must be lifted after the put-back")
}
if r.markerActive() {
t.Fatal("the marker must let go of the app after its put-back")
}
if g1.ResumePutBack("immich") != true || len(r.puts) != 1 {
t.Fatalf("a second resume must pour nothing, puts=%v", r.puts)
}
// the household writes on the drive, then any later controller start: nothing is poured back
householdWrite := filepath.Join(r.drive, "felhom-data", "new-photo-after-the-put-back")
mustWrite(t, householdWrite, "x")
if rec2 := r.guard().Recover(); rec2 != nil {
t.Fatalf("a later start must find nothing to recover, got %+v", rec2)
}
if len(r.puts) != 1 {
t.Fatalf("a later start poured the copy again over the household's writes: puts=%v", r.puts)
}
if _, err := os.Stat(householdWrite); err != nil {
t.Fatal("the household's newest file must still be there")
}
}
// Every start path is refused while the put-back waits: they all read the ONE hold store. The night chain and the
// household's buttons read it through RestoreHoldFor (isHeld, the router's restoreHoldFor, the update guards); the
// boot sweep and Recover through driveStartGate's GetRestoreHold; the web paths (boot recreate, drive return, media
// restart) through appHeld. The web and main.go wiring is pinned in their own packages.
func TestR1251_TheWaitingHoldRefusesEveryStartPath(t *testing.T) {
r := newR1251Rig(t)
r.unbind(t)
r.guard().Recover()
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
held, why := m.RestoreHoldForLang("immich", "hu")
if !held || !strings.Contains(why, "meghajt") || !strings.Contains(why, "immich") {
t.Fatalf("the household's sentence must name the app and the drive: held=%v %q", held, why)
}
if _, en := m.RestoreHoldForLang("immich", "en"); !strings.Contains(en, "drive is back") {
t.Fatalf("the English sentence: %q", en)
}
if !m.isHeld("immich") {
t.Fatal("the night chain's isHeld must see the waiting put-back")
}
// the gated starter (the boot sweep's and Recover's path) refuses too
if err := (r1251Starter{r}).StartStack("immich"); !errors.Is(err, ErrStartRefused) {
t.Fatalf("the gated start must be refused while held, got %v", err)
}
// a removal ends the install the copy belongs to: the waiting hold goes with it (a later install must not wait)
if cleared, _ := r.sett.ClearUpdateHold("immich"); !cleared {
t.Fatal("a removal must clear the waiting put-back hold")
}
}
// H2: a put-back that fails at start leaves the app HELD (the D20 hold, „stopped half-way"), never stopped-and-unheld;
// the copy stays and the marker lets go (the hold owns the app).
func TestR1251_AFailedPutBackHoldsTheApp(t *testing.T) {
r := newR1251Rig(t)
r.putErr = errors.New("tar: short read")
rec := r.guard().Recover()
h, held := r.sett.GetRestoreHold("immich")
if !held || h.Reason != settings.HoldReasonRestoreMixed {
t.Fatalf("a failed put-back must leave a restore_mixed hold, got %+v held=%v", h, held)
}
if rec == nil || len(rec.Held) != 1 || !rec.Alarming() || len(r.starts) != 0 {
t.Fatalf("held, alarming, not started: rec=%+v starts=%v", rec, r.starts)
}
if !strings.Contains(h.ReplayError, "short read") || h.SafetyDump != r.undo {
t.Fatalf("the hold must carry the cause and the copy: %+v", h)
}
if _, err := os.Stat(r.undo); err != nil {
t.Fatal("the copy must stay on disk for support")
}
if r.markerActive() {
t.Fatal("the hold owns the app now — the marker must let go, so no later start pours an old copy over it")
}
// the household's Start, the boot recreate and the update are all refused by that hold
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
if held, why := m.RestoreHoldForLang("immich", "hu"); !held || !strings.Contains(why, "immich") {
t.Fatalf("the D20 sentence: held=%v %q", held, why)
}
}
// A failed resume (the drive came back, the put-back broke) holds the app the same way.
func TestR1251_AFailedResumeHoldsTheApp(t *testing.T) {
r := newR1251Rig(t)
r.unbind(t)
g := r.guard()
g.Recover()
r.bind(t)
r.putErr = errors.New("disk full")
if g.ResumePutBack("immich") {
t.Fatal("a failed put-back must not allow the start")
}
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
t.Fatalf("a failed resume must hold for support, got %+v", h)
}
if r.markerActive() {
t.Fatal("the marker must let go after the hold took over")
}
}
// A copy that vanished while its drive is BOUND, during a waiting put-back, cannot be poured: held for support.
func TestR1251_ACopyGoneWhileItsDriveIsBoundHoldsTheApp(t *testing.T) {
r := newR1251Rig(t)
r.unbind(t)
g := r.guard()
g.Recover()
r.bind(t)
if err := os.RemoveAll(r.undo); err != nil {
t.Fatal(err)
}
if g.ResumePutBack("immich") {
t.Fatal("a vanished copy must not allow the start")
}
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
t.Fatalf("held for support, got %+v", h)
}
}
// H7: Stop pressed during the restore — the copy is still poured back after the cut; only the start is skipped.
func TestR1251_StopPressedDuringTheRestoreStillPutsBack(t *testing.T) {
r := newR1251Rig(t)
r.stopped["immich"] = true
rec := r.guard().Recover()
if strings.Join(r.puts, ",") != "immich" {
t.Fatalf("the put-back must run even though the household pressed Stop, puts=%v rec=%+v", r.puts, rec)
}
if len(r.starts) != 0 {
t.Fatalf("the household stopped it — it must not start, starts=%v", r.starts)
}
if r.markerActive() {
t.Fatal("nothing is owed any more — the marker must clear")
}
}
// The drive returns later (not at boot): the sweep puts back, then starts through the gated starter — unless the
// household stopped the app.
func TestR1251_ADriveThatReturnsLaterIsPutBackThenStarted(t *testing.T) {
for _, stopped := range []bool{false, true} {
r := newR1251Rig(t)
r.unbind(t)
g := r.guard()
g.Recover()
if got := g.ResumeWaitingPutBacks(); len(got) != 0 || len(r.puts) != 0 {
t.Fatalf("drive still absent: nothing may happen, started=%v puts=%v", got, r.puts)
}
r.bind(t)
r.stopped["immich"] = stopped
got := g.ResumeWaitingPutBacks()
if len(r.puts) != 1 {
t.Fatalf("stopped=%v: the put-back must run once, puts=%v", stopped, r.puts)
}
if want := !stopped; (len(got) == 1) != want || (len(r.starts) == 1) != want {
t.Fatalf("stopped=%v: started=%v starts=%v", stopped, got, r.starts)
}
}
}
// A later restore that completes drops the waiting put-back: the household chose that restore's data.
func TestR1251_ACompletedRestoreDropsTheWaitingPutBack(t *testing.T) {
r := newR1251Rig(t)
r.unbind(t)
r.guard().Recover()
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
m.clearUpdateHoldAfterRestore("immich")
if _, held := r.sett.GetRestoreHold("immich"); held {
t.Fatal("a completed restore must drop the waiting put-back hold")
}
}
@@ -653,6 +653,16 @@ func (m *Manager) clearUpdateHoldAfterRestore(stackName string) {
return
}
h, ok := m.settings.GetRestoreHold(stackName)
if ok && h.Reason == settings.HoldReasonPutBackPending {
// R-1251: a restore that completed replaced the data a waiting put-back would have poured back — the household
// chose this restore, so the old copy must never be poured over it when its drive returns. The copy stays.
if _, err := m.settings.ClearRestoreHold(stackName); err != nil {
m.logger.Printf("[ERROR] [backup] %s was restored, but its waiting put-back hold could not be cleared: %v", stackName, err)
return
}
m.logger.Printf("[INFO] [backup] %s: restore completed — the waiting put-back (copy %s, set %s) is DROPPED; the copy stays on disk", stackName, h.SafetyDump, h.At)
return
}
if !ok || h.Reason != settings.HoldReasonUpdateFailed {
return
}