R-1251 (P1): a pending put-back blocks every start — a cut restore's copy on an unbound drive waits (a restore hold), is poured back when the drive binds before any start; a failed put-back holds the app (H2); Stop during a restore still puts back (H7). Commit only, unreleased.
gates / gates (push) Successful in 1m18s
gates / gates (push) Successful in 1m18s
This commit is contained in:
@@ -133,6 +133,10 @@ type AppStopGuard struct {
|
||||
// putBacker (D22, R-1195) pours a restore's undo copy back before Recover restarts the app. nil = no put-back
|
||||
// (the pre-D22 restart, unchanged).
|
||||
putBacker InterruptedRestorePutBacker
|
||||
// holds and undoReachable (R-1251, appstop_putback_wait.go): where a waiting or failed put-back is HELD, and
|
||||
// whether the drive a copy lives on is bound. nil = the pre-R-1251 behaviour.
|
||||
holds AppStopHolds
|
||||
undoReachable func(stack, dir string) bool
|
||||
|
||||
// suppressed (R-330) is the in-memory "the alarm must not fire for these, we are holding them"
|
||||
// set. It is deliberately NOT persisted: after a restart the guard no longer holds anything —
|
||||
@@ -162,6 +166,11 @@ type AppStopRecovery struct {
|
||||
// PutBack are apps whose interrupted restore was undone from its copy before the restart (D22, R-1195): their
|
||||
// data is as it was before the restore.
|
||||
PutBack []string
|
||||
// Waiting (R-1251) are apps whose restore copy is on a drive that is not bound: HELD, not started, put back first
|
||||
// when the drive binds (ResumePutBack). Like Refused, the holder owns them and they do not alarm.
|
||||
Waiting []string
|
||||
// Held (R-1251 H2) are apps whose put-back FAILED and that are now held for support (the D20 hold). They alarm.
|
||||
Held []string
|
||||
}
|
||||
|
||||
// Alarming reports whether this recovery is worth paging an operator about. A recovery that only
|
||||
@@ -171,7 +180,7 @@ func (r *AppStopRecovery) Alarming() bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
return len(r.Failed) > 0 || len(r.Restarted) > 0
|
||||
return len(r.Failed) > 0 || len(r.Held) > 0 || len(r.Restarted) > 0
|
||||
}
|
||||
|
||||
// Message is the operator-facing headline for an interrupted operation.
|
||||
@@ -179,14 +188,22 @@ func (r *AppStopRecovery) Message() string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
if len(r.Failed) > 0 {
|
||||
if len(r.Failed) > 0 || len(r.Held) > 0 {
|
||||
bad := len(r.Failed) + len(r.Held)
|
||||
m := fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted",
|
||||
r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed))
|
||||
r.Reason.humanReason(), bad, len(r.Restarted)+bad)
|
||||
if len(r.Held) > 0 {
|
||||
m += fmt.Sprintf(" (%d could not be put back and are HELD for support)", len(r.Held))
|
||||
}
|
||||
if len(r.Refused) > 0 {
|
||||
m += fmt.Sprintf(" (a further %d are held by an absent drive and are not counted as failures)", len(r.Refused))
|
||||
}
|
||||
return m
|
||||
}
|
||||
if len(r.Waiting) > 0 && len(r.Restarted) == 0 {
|
||||
return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are HELD until their data drive is bound: the copy taken before the change is on it and is put back first",
|
||||
r.Reason.humanReason(), len(r.Waiting))
|
||||
}
|
||||
if len(r.Refused) > 0 && len(r.Restarted) == 0 {
|
||||
return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are left stopped and HELD: their data drive is not available, so the drive gate restarts them when it returns",
|
||||
r.Reason.humanReason(), len(r.Refused))
|
||||
@@ -215,6 +232,12 @@ func (r *AppStopRecovery) Detail() string {
|
||||
if len(r.PutBack) > 0 {
|
||||
d += fmt.Sprintf(" put_back=%v", r.PutBack)
|
||||
}
|
||||
if len(r.Waiting) > 0 {
|
||||
d += fmt.Sprintf(" put_back_waiting_for_drive=%v", r.Waiting)
|
||||
}
|
||||
if len(r.Held) > 0 {
|
||||
d += fmt.Sprintf(" put_back_failed_held=%v", r.Held)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
@@ -352,28 +375,48 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
|
||||
|
||||
res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt}
|
||||
hs, _ := g.starter.(interface{ WantsStopped(string) bool })
|
||||
var usedUndo []string // apps whose copy was poured back: the marker must not name it again
|
||||
for _, name := range m.Stacks {
|
||||
if hs != nil && hs.WantsStopped(name) {
|
||||
// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down.
|
||||
// It is owed nothing; the marker still clears below.
|
||||
g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
|
||||
continue
|
||||
}
|
||||
// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down — it is owed
|
||||
// no restart. R-1251 H7: it is still owed its put-back, so the stop is honoured AFTER the copy is poured back.
|
||||
wantsStopped := hs != nil && hs.WantsStopped(name)
|
||||
if dir := m.Undo[name]; dir != "" && g.putBacker != nil {
|
||||
// D22 (R-1195): the restore was cut after its copy of the live data was taken — the app may be on the
|
||||
// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery
|
||||
// (the marker is kept, so the next startup tries again) and the app is not started on unknown data.
|
||||
if _, statErr := os.Stat(dir); statErr != nil {
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
|
||||
} else if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
|
||||
res.Failed = append(res.Failed, name)
|
||||
// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery and the
|
||||
// app is not started on unknown data.
|
||||
state, statErr := g.copyState(name, dir)
|
||||
switch state {
|
||||
case undoWaiting:
|
||||
// R-1251: the copy is on a drive that is not bound yet. It is not „gone": the app is HELD (every start
|
||||
// path honours the hold) and the copy is poured back when the drive binds (ResumePutBack).
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s cannot be reached yet (%v) — NOT starting it; the put-back waits for its drive", name, dir, statErr)
|
||||
g.holdPutBackPending(name, dir)
|
||||
res.Waiting = append(res.Waiting, name)
|
||||
continue
|
||||
} else {
|
||||
case undoGone:
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
|
||||
default:
|
||||
if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
|
||||
// R-1251 H2: HELD (the D20 hold), never stopped-and-unheld. Without a hold store the old way stays:
|
||||
// Failed, and the marker is kept so the next start tries again.
|
||||
if g.holdPutBackFailed(name, dir, err) {
|
||||
res.Held = append(res.Held, name)
|
||||
} else {
|
||||
res.Failed = append(res.Failed, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %s's interrupted restore is undone — its data is as before the restore (R-1195)", name)
|
||||
g.clearPutBackPending(name)
|
||||
res.PutBack = append(res.PutBack, name)
|
||||
usedUndo = append(usedUndo, name)
|
||||
}
|
||||
}
|
||||
if wantsStopped {
|
||||
g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
|
||||
continue
|
||||
}
|
||||
if err := g.starter.StartStack(name); err != nil {
|
||||
// R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be
|
||||
// started (an absent data drive), so the app is left down deliberately and the holder
|
||||
@@ -395,6 +438,8 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
|
||||
sort.Strings(res.Refused)
|
||||
sort.Strings(res.Restarted)
|
||||
sort.Strings(res.PutBack)
|
||||
sort.Strings(res.Waiting)
|
||||
sort.Strings(res.Held)
|
||||
|
||||
// The marker is kept for BOTH unfinished outcomes, for the same reason and with different
|
||||
// urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished
|
||||
@@ -403,11 +448,16 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
|
||||
if len(res.Failed) > 0 {
|
||||
g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v",
|
||||
len(res.Failed), res.Failed)
|
||||
g.dropFromMarker(res.Held...)
|
||||
g.forgetUndo(usedUndo...)
|
||||
return res
|
||||
}
|
||||
if len(res.Refused) > 0 {
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent) — KEEPING the marker; this is the gate working, not a fault: %v",
|
||||
len(res.Refused), res.Refused)
|
||||
if len(res.Refused) > 0 || len(res.Waiting) > 0 {
|
||||
g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent: %v; put-back waiting for its drive: %v) — KEEPING the marker; this is the gate working, not a fault",
|
||||
len(res.Refused)+len(res.Waiting), res.Refused, res.Waiting)
|
||||
// The held (H2) apps and the used copies leave the marker; the rest of it stays for the next start.
|
||||
g.dropFromMarker(res.Held...)
|
||||
g.forgetUndo(usedUndo...)
|
||||
return res
|
||||
}
|
||||
g.End()
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── R-1251: a pending put-back blocks every start (review round 2, HUNT-1 H1) ─────────────────────────────────────
|
||||
//
|
||||
// D22 pours a cut restore's copy back at start (restore_crash_putback.go). For a DRIVE-backed app the copy lives on
|
||||
// the app's drive (restoreUndoDir under the drive's namespace), and at start the drive is often not bound yet. Until
|
||||
// this file Recover read the missing folder as „gone", went on to the gated start (refused: no drive) and kept the
|
||||
// marker; the drive then bound, the boot recreate started the app on the cut restore's data (it asked only „held?"),
|
||||
// the household wrote, and the NEXT controller start found the copy and poured it over everything written since.
|
||||
//
|
||||
// Now:
|
||||
// - a copy that cannot be seen while its drive is not bound is WAITING, never gone. Recover writes a restore hold
|
||||
// (settings.HoldReasonPutBackPending, the copy's folder in SafetyDump) and starts nothing. The hold is the ONE
|
||||
// storage every start path already honours — the household's Start/Restart/Update, the boot recreate, the drive
|
||||
// return, the media restart, the boot sweep, the night chain, the update leg — so no path needs its own check;
|
||||
// - when the drive binds, ResumePutBack pours the copy back FIRST and lifts the hold; only then may the app start;
|
||||
// - a put-back that FAILS (at start or on resume) leaves the app HELD with the D20 hold (HoldReasonRestoreMixed, the
|
||||
// household's „stopped half-way" sentence) and the copy on disk — never stopped-and-unheld, which Start or the boot
|
||||
// recreate would run and a later start would pour an old copy over (H2). The marker then lets go of that app: the
|
||||
// hold owns it, exactly as holdAppAfterMixedRestore does in-process;
|
||||
// - Stop pressed during the restore no longer skips the put-back after a cut (H7): the data is put back, the app
|
||||
// stays stopped as the household asked.
|
||||
//
|
||||
// The hold — not the marker — is the durable record of a waiting put-back, because the marker has one slot and the
|
||||
// next operation's Begin replaces it (R-1253).
|
||||
//
|
||||
// Pinned by r1251_putback_waits_test.go.
|
||||
|
||||
// AppStopHolds is the hold store the guard writes (settings.Settings in production).
|
||||
type AppStopHolds interface {
|
||||
SetRestoreHold(settings.RestoreHold) error
|
||||
GetRestoreHold(stack string) (settings.RestoreHold, bool)
|
||||
ClearRestoreHold(stack string) (bool, error)
|
||||
ListRestoreHolds() []settings.RestoreHold
|
||||
}
|
||||
|
||||
// SetHolds wires the hold store. INIT-ONLY, before Recover. nil = the pre-R-1251 behaviour (no hold is written).
|
||||
func (g *AppStopGuard) SetHolds(h AppStopHolds) {
|
||||
if g != nil {
|
||||
g.holds = h
|
||||
}
|
||||
}
|
||||
|
||||
// SetUndoReachable wires the question „is the drive this app's restore copy lives on bound right now?" (main.go:
|
||||
// the app's HDD_PATH is a live mountpoint). nil = every folder counts as reachable, so a missing copy is „gone".
|
||||
func (g *AppStopGuard) SetUndoReachable(f func(stack, dir string) bool) {
|
||||
if g != nil {
|
||||
g.undoReachable = f
|
||||
}
|
||||
}
|
||||
|
||||
type undoCopyState int
|
||||
|
||||
const (
|
||||
undoPresent undoCopyState = iota
|
||||
undoGone // its drive is bound and the folder is not there
|
||||
undoWaiting // its drive is not bound (or the folder cannot be read): it may come back
|
||||
)
|
||||
|
||||
func (s undoCopyState) String() string {
|
||||
switch s {
|
||||
case undoPresent:
|
||||
return "present"
|
||||
case undoGone:
|
||||
return "gone"
|
||||
default:
|
||||
return "waiting"
|
||||
}
|
||||
}
|
||||
|
||||
// copyState decides present / gone / waiting for one recorded copy.
|
||||
func (g *AppStopGuard) copyState(stack, dir string) (undoCopyState, error) {
|
||||
_, err := os.Stat(dir)
|
||||
if err == nil {
|
||||
return undoPresent, nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return undoWaiting, err // an unreadable folder is not proof that it is gone
|
||||
}
|
||||
if g.undoReachable != nil && !g.undoReachable(stack, dir) {
|
||||
return undoWaiting, err
|
||||
}
|
||||
return undoGone, err
|
||||
}
|
||||
|
||||
// holdPutBackPending writes the „waiting for the drive" hold. Returns whether the app is now held by SOME hold.
|
||||
func (g *AppStopGuard) holdPutBackPending(stack, dir string) bool {
|
||||
if g.holds == nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: its restore copy %s waits for its drive, but no hold store is wired — nothing durable refuses a start (R-1251)", stack, dir)
|
||||
return false
|
||||
}
|
||||
if h, ok := g.holds.GetRestoreHold(stack); ok && h.Reason != settings.HoldReasonPutBackPending {
|
||||
// Another hold already refuses every start, and its sentence names the harder problem. The marker keeps the
|
||||
// copy's folder, so the next start puts it back once the drive is there.
|
||||
g.logger.Printf("[WARN] [appstop] %s: its restore copy %s waits for its drive; the app is already held (%q, set %s) — that hold stays", stack, dir, h.Reason, h.At)
|
||||
return true
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending, SafetyDump: dir}
|
||||
if err := g.holds.SetRestoreHold(h); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the waiting put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
|
||||
return false
|
||||
}
|
||||
g.logger.Printf("[WARN] [appstop] %s: the cut restore's copy %s is on a drive that is not bound — the app is HELD until the copy is put back (R-1251)", stack, dir)
|
||||
return true
|
||||
}
|
||||
|
||||
// holdPutBackFailed writes the D20 hold for a put-back that could not be done (H2). Returns whether it was written.
|
||||
func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
|
||||
if g.holds == nil {
|
||||
return false
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
|
||||
ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
|
||||
if err := g.holds.SetRestoreHold(h); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
|
||||
return false
|
||||
}
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the interrupted restore could NOT be put back (%v) — the app is HELD for support, the copy stays at %s (R-1251 H2)", stack, cause, dir)
|
||||
return true
|
||||
}
|
||||
|
||||
// clearPutBackPending lifts this app's waiting-put-back hold, and only that kind.
|
||||
func (g *AppStopGuard) clearPutBackPending(stack string) {
|
||||
if g.holds == nil {
|
||||
return
|
||||
}
|
||||
if h, ok := g.holds.GetRestoreHold(stack); !ok || h.Reason != settings.HoldReasonPutBackPending {
|
||||
return
|
||||
}
|
||||
if _, err := g.holds.ClearRestoreHold(stack); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: the put-back ran, but its hold could not be cleared: %v — the app stays held", stack, err)
|
||||
}
|
||||
}
|
||||
|
||||
// dropFromMarker lets the active marker forget these apps (and their copies): a hold now owns them, or their put-back
|
||||
// is done. A marker left with no app is cleared. A marker that belongs to another operation (it does not hold them)
|
||||
// is not touched.
|
||||
func (g *AppStopGuard) dropFromMarker(names ...string) {
|
||||
if g == nil || g.path == "" || len(names) == 0 {
|
||||
return
|
||||
}
|
||||
m, ok := g.read()
|
||||
if !ok || !m.Active {
|
||||
return
|
||||
}
|
||||
drop := map[string]bool{}
|
||||
for _, n := range names {
|
||||
drop[n] = true
|
||||
}
|
||||
var keep []string
|
||||
for _, s := range m.Stacks {
|
||||
if !drop[s] {
|
||||
keep = append(keep, s)
|
||||
}
|
||||
}
|
||||
if len(keep) == len(m.Stacks) {
|
||||
return
|
||||
}
|
||||
if len(keep) == 0 {
|
||||
if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) {
|
||||
g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v", g.path, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
m.Stacks = keep
|
||||
for n := range drop {
|
||||
delete(m.Undo, n)
|
||||
}
|
||||
if err := g.write(m); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without %v: %v", names, err)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetUndo removes these apps' copy folders from the active marker (their put-back is done; the copy is removed),
|
||||
// so a later start never tries to pour back a folder that no longer holds the app's data.
|
||||
func (g *AppStopGuard) forgetUndo(names ...string) {
|
||||
if g == nil || g.path == "" || len(names) == 0 {
|
||||
return
|
||||
}
|
||||
m, ok := g.read()
|
||||
if !ok || !m.Active || len(m.Undo) == 0 {
|
||||
return
|
||||
}
|
||||
changed := false
|
||||
for _, n := range names {
|
||||
if _, ok := m.Undo[n]; ok {
|
||||
delete(m.Undo, n)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
if err := g.write(m); err != nil {
|
||||
g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without the used copies of %v: %v", names, err)
|
||||
}
|
||||
}
|
||||
|
||||
// resumeMu serialises ResumePutBack: the boot recreate and the drive-return sweep can reach the same app at once, and
|
||||
// a copy must be poured back exactly once.
|
||||
var resumeMu sync.Mutex
|
||||
|
||||
// ResumePutBack runs a put-back that was WAITING for its drive. It returns true when nothing is owed for stack any
|
||||
// more — there was no waiting put-back, or it ran now — and the caller may go on to start the app (its other holds
|
||||
// still apply). false: the app stays held (the drive is still not bound, or the put-back failed and the app is now
|
||||
// held for support). Nil-safe.
|
||||
func (g *AppStopGuard) ResumePutBack(stack string) bool {
|
||||
if g == nil || g.holds == nil {
|
||||
return true
|
||||
}
|
||||
resumeMu.Lock()
|
||||
defer resumeMu.Unlock()
|
||||
h, ok := g.holds.GetRestoreHold(stack)
|
||||
if !ok || h.Reason != settings.HoldReasonPutBackPending {
|
||||
return true
|
||||
}
|
||||
dir := h.SafetyDump
|
||||
if m, ok := g.read(); ok && m.Active && m.Undo[stack] != "" {
|
||||
dir = m.Undo[stack]
|
||||
}
|
||||
if dir == "" || g.putBacker == nil {
|
||||
g.logger.Printf("[ERROR] [appstop] %s: a put-back waits, but its copy is not named (or no put-back is wired) — the app stays HELD", stack)
|
||||
return false
|
||||
}
|
||||
state, serr := g.copyState(stack, dir)
|
||||
switch state {
|
||||
case undoWaiting:
|
||||
g.logger.Printf("[INFO] [appstop] %s: the restore copy %s is still not reachable (%v) — the app stays held until its drive is bound", stack, dir, serr)
|
||||
return false
|
||||
case undoGone:
|
||||
g.holdPutBackFailed(stack, dir, fmt.Errorf("the restore copy %s is gone while its drive is bound", dir))
|
||||
g.dropFromMarker(stack)
|
||||
return false
|
||||
}
|
||||
if err := g.putBacker.PutBackInterruptedRestore(stack, dir); err != nil {
|
||||
if g.holdPutBackFailed(stack, dir, err) {
|
||||
g.dropFromMarker(stack)
|
||||
}
|
||||
return false
|
||||
}
|
||||
g.clearPutBackPending(stack)
|
||||
g.dropFromMarker(stack)
|
||||
g.logger.Printf("[WARN] [appstop] %s: the drive is bound — the interrupted restore is put back from %s BEFORE the app starts; its data is as before the restore (R-1251)", stack, dir)
|
||||
return true
|
||||
}
|
||||
|
||||
// ResumeWaitingPutBacks runs every waiting put-back whose drive is bound now, and starts each app afterwards through
|
||||
// the guard's gated starter — unless the household stopped it. Called at start (after Recover) and when a drive
|
||||
// returns. Returns the apps started.
|
||||
func (g *AppStopGuard) ResumeWaitingPutBacks() []string {
|
||||
if g == nil || g.holds == nil {
|
||||
return nil
|
||||
}
|
||||
var started []string
|
||||
for _, h := range g.holds.ListRestoreHolds() {
|
||||
if h.Reason != settings.HoldReasonPutBackPending {
|
||||
continue
|
||||
}
|
||||
if !g.ResumePutBack(h.Stack) {
|
||||
continue
|
||||
}
|
||||
if ws, ok := g.starter.(interface{ WantsStopped(string) bool }); ok && ws.WantsStopped(h.Stack) {
|
||||
g.logger.Printf("[INFO] [appstop] %s: put back; NOT starting it — the household stopped it", h.Stack)
|
||||
continue
|
||||
}
|
||||
if g.starter == nil {
|
||||
continue
|
||||
}
|
||||
if err := g.starter.StartStack(h.Stack); err != nil {
|
||||
g.logger.Printf("[WARN] [appstop] %s: put back, but the start was not done: %v", h.Stack, err)
|
||||
continue
|
||||
}
|
||||
started = append(started, h.Stack)
|
||||
}
|
||||
return started
|
||||
}
|
||||
@@ -369,6 +369,9 @@ func (m *Manager) RestoreHoldForLang(stack, lang string) (bool, string) {
|
||||
if t, err := time.Parse(time.RFC3339, h.At); err == nil {
|
||||
when = t.Format("2006-01-02 15:04")
|
||||
}
|
||||
if h.Reason == settings.HoldReasonPutBackPending { // R-1251: the put-back waits for the copy's drive
|
||||
return true, util.Text(lang, "hold.putback_pending", stack)
|
||||
}
|
||||
if h.Reason == settings.HoldReasonRestoreMixed { // R-893
|
||||
return true, util.Text(lang, "note.reconstitute.held_mixed", stack, when)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
package backup
|
||||
|
||||
// R-1251 (review round 2, HUNT-1 H1, H2, H7): a pending put-back blocks every start. The first test is the review's
|
||||
// lean-clone proof (audits/review-round-2-2026-10-24/proofs/H1-drive-cut-restore/) turned round: the same cut, the same
|
||||
// unbound drive, and now the app is HELD until the copy is poured back, and a later start pours nothing.
|
||||
//
|
||||
// RED-PROOF (2026-10-24, each seen failing on main 2449223 before appstop_putback_wait.go):
|
||||
// - TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart: „boot 1 must HOLD the app as waiting" —
|
||||
// Recover read the copy as gone and only REFUSED the start, with no hold;
|
||||
// - TestR1251_AFailedPutBackHoldsTheApp: „a failed put-back must leave a restore_mixed hold" — none was written;
|
||||
// - TestR1251_StopPressedDuringTheRestoreStillPutsBack: „the put-back must run" — R-721's skip came first.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// r1251Rig is one app (immich) whose restore was cut after its copy was recorded, the copy on the app's drive.
|
||||
type r1251Rig struct {
|
||||
marker, drive, undo string
|
||||
sett *settings.Settings
|
||||
puts []string
|
||||
starts []string
|
||||
stopped map[string]bool
|
||||
putErr error
|
||||
}
|
||||
|
||||
func newR1251Rig(t *testing.T) *r1251Rig {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
r := &r1251Rig{
|
||||
marker: filepath.Join(root, "appstop-state.json"),
|
||||
drive: filepath.Join(root, "drive"),
|
||||
stopped: map[string]bool{},
|
||||
}
|
||||
r.undo = filepath.Join(r.drive, "felhom-data", "backups", "restore-undo", "immich", "20261024T120000Z")
|
||||
mustWrite(t, filepath.Join(r.undo, "undo.json"), `{"stack":"immich","copied":["immich_pgdata"]}`)
|
||||
sett, err := settings.Load(filepath.Join(root, "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.sett = sett
|
||||
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
|
||||
if err := g.Begin("unit-restore:immich", ReasonUnitRestore, []string{"immich"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.SetUndo("immich", r.undo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r // the power cut: no End
|
||||
}
|
||||
|
||||
func (r *r1251Rig) unbind(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := os.Rename(r.drive, r.drive+".unbound"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *r1251Rig) bind(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := os.Rename(r.drive+".unbound", r.drive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *r1251Rig) driveBound() bool { _, err := os.Stat(r.drive); return err == nil }
|
||||
|
||||
// r1251Starter is the gated starter: it refuses while the drive is absent or a hold is set (as driveStartGate does).
|
||||
type r1251Starter struct{ r *r1251Rig }
|
||||
|
||||
func (s r1251Starter) StartStack(name string) error {
|
||||
if _, held := s.r.sett.GetRestoreHold(name); held {
|
||||
return ErrStartRefused
|
||||
}
|
||||
if !s.r.driveBound() {
|
||||
return ErrStartRefused
|
||||
}
|
||||
s.r.starts = append(s.r.starts, name)
|
||||
return nil
|
||||
}
|
||||
func (s r1251Starter) WantsStopped(name string) bool { return s.r.stopped[name] }
|
||||
|
||||
type r1251PutBacker struct{ r *r1251Rig }
|
||||
|
||||
func (p r1251PutBacker) PutBackInterruptedRestore(stack, dir string) error {
|
||||
if p.r.putErr != nil {
|
||||
return p.r.putErr
|
||||
}
|
||||
p.r.puts = append(p.r.puts, stack)
|
||||
return os.RemoveAll(dir) // the real put-back removes the used copy
|
||||
}
|
||||
|
||||
// guard is a fresh controller start over the rig's disk, wired as main.go wires it.
|
||||
func (r *r1251Rig) guard() *AppStopGuard {
|
||||
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
|
||||
g.SetStarter(r1251Starter{r})
|
||||
g.SetPutBacker(r1251PutBacker{r})
|
||||
g.SetHolds(r.sett)
|
||||
g.SetUndoReachable(func(string, string) bool { return r.driveBound() })
|
||||
return g
|
||||
}
|
||||
|
||||
func (r *r1251Rig) markerActive() bool {
|
||||
b, err := os.ReadFile(r.marker)
|
||||
return err == nil && strings.Contains(string(b), `"active": true`)
|
||||
}
|
||||
|
||||
func TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
|
||||
// boot 1: the drive is not bound yet
|
||||
r.unbind(t)
|
||||
g1 := r.guard()
|
||||
rec := g1.Recover()
|
||||
h, held := r.sett.GetRestoreHold("immich")
|
||||
if rec == nil || len(rec.Waiting) != 1 || !held || h.Reason != settings.HoldReasonPutBackPending {
|
||||
t.Fatalf("boot 1 must HOLD the app as waiting for its drive: rec=%+v hold=%+v held=%v", rec, h, held)
|
||||
}
|
||||
if h.SafetyDump != r.undo {
|
||||
t.Fatalf("the hold must name the copy, got %q", h.SafetyDump)
|
||||
}
|
||||
if len(r.starts) != 0 || len(r.puts) != 0 || rec.Alarming() {
|
||||
t.Fatalf("nothing may start or be poured while the drive is absent: starts=%v puts=%v alarming=%v", r.starts, r.puts, rec.Alarming())
|
||||
}
|
||||
if !r.markerActive() {
|
||||
t.Fatal("the marker must be kept while the put-back waits")
|
||||
}
|
||||
// a resume while the drive is still absent does nothing and keeps the hold
|
||||
if g1.ResumePutBack("immich") {
|
||||
t.Fatal("ResumePutBack must say still-held while the copy cannot be reached")
|
||||
}
|
||||
if _, still := r.sett.GetRestoreHold("immich"); !still {
|
||||
t.Fatal("the hold must stay while the drive is absent")
|
||||
}
|
||||
|
||||
// the drive binds: the boot recreate (and the drive-return sweep) call ResumePutBack before they start the app
|
||||
r.bind(t)
|
||||
if !g1.ResumePutBack("immich") {
|
||||
t.Fatal("with the drive bound the put-back must run and allow the start")
|
||||
}
|
||||
if strings.Join(r.puts, ",") != "immich" {
|
||||
t.Fatalf("the copy must be poured back exactly once, puts=%v", r.puts)
|
||||
}
|
||||
if _, still := r.sett.GetRestoreHold("immich"); still {
|
||||
t.Fatal("the waiting hold must be lifted after the put-back")
|
||||
}
|
||||
if r.markerActive() {
|
||||
t.Fatal("the marker must let go of the app after its put-back")
|
||||
}
|
||||
if g1.ResumePutBack("immich") != true || len(r.puts) != 1 {
|
||||
t.Fatalf("a second resume must pour nothing, puts=%v", r.puts)
|
||||
}
|
||||
|
||||
// the household writes on the drive, then any later controller start: nothing is poured back
|
||||
householdWrite := filepath.Join(r.drive, "felhom-data", "new-photo-after-the-put-back")
|
||||
mustWrite(t, householdWrite, "x")
|
||||
if rec2 := r.guard().Recover(); rec2 != nil {
|
||||
t.Fatalf("a later start must find nothing to recover, got %+v", rec2)
|
||||
}
|
||||
if len(r.puts) != 1 {
|
||||
t.Fatalf("a later start poured the copy again over the household's writes: puts=%v", r.puts)
|
||||
}
|
||||
if _, err := os.Stat(householdWrite); err != nil {
|
||||
t.Fatal("the household's newest file must still be there")
|
||||
}
|
||||
}
|
||||
|
||||
// Every start path is refused while the put-back waits: they all read the ONE hold store. The night chain and the
|
||||
// household's buttons read it through RestoreHoldFor (isHeld, the router's restoreHoldFor, the update guards); the
|
||||
// boot sweep and Recover through driveStartGate's GetRestoreHold; the web paths (boot recreate, drive return, media
|
||||
// restart) through appHeld. The web and main.go wiring is pinned in their own packages.
|
||||
func TestR1251_TheWaitingHoldRefusesEveryStartPath(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.unbind(t)
|
||||
r.guard().Recover()
|
||||
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
||||
held, why := m.RestoreHoldForLang("immich", "hu")
|
||||
if !held || !strings.Contains(why, "meghajt") || !strings.Contains(why, "immich") {
|
||||
t.Fatalf("the household's sentence must name the app and the drive: held=%v %q", held, why)
|
||||
}
|
||||
if _, en := m.RestoreHoldForLang("immich", "en"); !strings.Contains(en, "drive is back") {
|
||||
t.Fatalf("the English sentence: %q", en)
|
||||
}
|
||||
if !m.isHeld("immich") {
|
||||
t.Fatal("the night chain's isHeld must see the waiting put-back")
|
||||
}
|
||||
// the gated starter (the boot sweep's and Recover's path) refuses too
|
||||
if err := (r1251Starter{r}).StartStack("immich"); !errors.Is(err, ErrStartRefused) {
|
||||
t.Fatalf("the gated start must be refused while held, got %v", err)
|
||||
}
|
||||
// a removal ends the install the copy belongs to: the waiting hold goes with it (a later install must not wait)
|
||||
if cleared, _ := r.sett.ClearUpdateHold("immich"); !cleared {
|
||||
t.Fatal("a removal must clear the waiting put-back hold")
|
||||
}
|
||||
}
|
||||
|
||||
// H2: a put-back that fails at start leaves the app HELD (the D20 hold, „stopped half-way"), never stopped-and-unheld;
|
||||
// the copy stays and the marker lets go (the hold owns the app).
|
||||
func TestR1251_AFailedPutBackHoldsTheApp(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.putErr = errors.New("tar: short read")
|
||||
rec := r.guard().Recover()
|
||||
h, held := r.sett.GetRestoreHold("immich")
|
||||
if !held || h.Reason != settings.HoldReasonRestoreMixed {
|
||||
t.Fatalf("a failed put-back must leave a restore_mixed hold, got %+v held=%v", h, held)
|
||||
}
|
||||
if rec == nil || len(rec.Held) != 1 || !rec.Alarming() || len(r.starts) != 0 {
|
||||
t.Fatalf("held, alarming, not started: rec=%+v starts=%v", rec, r.starts)
|
||||
}
|
||||
if !strings.Contains(h.ReplayError, "short read") || h.SafetyDump != r.undo {
|
||||
t.Fatalf("the hold must carry the cause and the copy: %+v", h)
|
||||
}
|
||||
if _, err := os.Stat(r.undo); err != nil {
|
||||
t.Fatal("the copy must stay on disk for support")
|
||||
}
|
||||
if r.markerActive() {
|
||||
t.Fatal("the hold owns the app now — the marker must let go, so no later start pours an old copy over it")
|
||||
}
|
||||
// the household's Start, the boot recreate and the update are all refused by that hold
|
||||
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
||||
if held, why := m.RestoreHoldForLang("immich", "hu"); !held || !strings.Contains(why, "immich") {
|
||||
t.Fatalf("the D20 sentence: held=%v %q", held, why)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed resume (the drive came back, the put-back broke) holds the app the same way.
|
||||
func TestR1251_AFailedResumeHoldsTheApp(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.unbind(t)
|
||||
g := r.guard()
|
||||
g.Recover()
|
||||
r.bind(t)
|
||||
r.putErr = errors.New("disk full")
|
||||
if g.ResumePutBack("immich") {
|
||||
t.Fatal("a failed put-back must not allow the start")
|
||||
}
|
||||
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
|
||||
t.Fatalf("a failed resume must hold for support, got %+v", h)
|
||||
}
|
||||
if r.markerActive() {
|
||||
t.Fatal("the marker must let go after the hold took over")
|
||||
}
|
||||
}
|
||||
|
||||
// A copy that vanished while its drive is BOUND, during a waiting put-back, cannot be poured: held for support.
|
||||
func TestR1251_ACopyGoneWhileItsDriveIsBoundHoldsTheApp(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.unbind(t)
|
||||
g := r.guard()
|
||||
g.Recover()
|
||||
r.bind(t)
|
||||
if err := os.RemoveAll(r.undo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.ResumePutBack("immich") {
|
||||
t.Fatal("a vanished copy must not allow the start")
|
||||
}
|
||||
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
|
||||
t.Fatalf("held for support, got %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// H7: Stop pressed during the restore — the copy is still poured back after the cut; only the start is skipped.
|
||||
func TestR1251_StopPressedDuringTheRestoreStillPutsBack(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.stopped["immich"] = true
|
||||
rec := r.guard().Recover()
|
||||
if strings.Join(r.puts, ",") != "immich" {
|
||||
t.Fatalf("the put-back must run even though the household pressed Stop, puts=%v rec=%+v", r.puts, rec)
|
||||
}
|
||||
if len(r.starts) != 0 {
|
||||
t.Fatalf("the household stopped it — it must not start, starts=%v", r.starts)
|
||||
}
|
||||
if r.markerActive() {
|
||||
t.Fatal("nothing is owed any more — the marker must clear")
|
||||
}
|
||||
}
|
||||
|
||||
// The drive returns later (not at boot): the sweep puts back, then starts through the gated starter — unless the
|
||||
// household stopped the app.
|
||||
func TestR1251_ADriveThatReturnsLaterIsPutBackThenStarted(t *testing.T) {
|
||||
for _, stopped := range []bool{false, true} {
|
||||
r := newR1251Rig(t)
|
||||
r.unbind(t)
|
||||
g := r.guard()
|
||||
g.Recover()
|
||||
if got := g.ResumeWaitingPutBacks(); len(got) != 0 || len(r.puts) != 0 {
|
||||
t.Fatalf("drive still absent: nothing may happen, started=%v puts=%v", got, r.puts)
|
||||
}
|
||||
r.bind(t)
|
||||
r.stopped["immich"] = stopped
|
||||
got := g.ResumeWaitingPutBacks()
|
||||
if len(r.puts) != 1 {
|
||||
t.Fatalf("stopped=%v: the put-back must run once, puts=%v", stopped, r.puts)
|
||||
}
|
||||
if want := !stopped; (len(got) == 1) != want || (len(r.starts) == 1) != want {
|
||||
t.Fatalf("stopped=%v: started=%v starts=%v", stopped, got, r.starts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A later restore that completes drops the waiting put-back: the household chose that restore's data.
|
||||
func TestR1251_ACompletedRestoreDropsTheWaitingPutBack(t *testing.T) {
|
||||
r := newR1251Rig(t)
|
||||
r.unbind(t)
|
||||
r.guard().Recover()
|
||||
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
||||
m.clearUpdateHoldAfterRestore("immich")
|
||||
if _, held := r.sett.GetRestoreHold("immich"); held {
|
||||
t.Fatal("a completed restore must drop the waiting put-back hold")
|
||||
}
|
||||
}
|
||||
@@ -653,6 +653,16 @@ func (m *Manager) clearUpdateHoldAfterRestore(stackName string) {
|
||||
return
|
||||
}
|
||||
h, ok := m.settings.GetRestoreHold(stackName)
|
||||
if ok && h.Reason == settings.HoldReasonPutBackPending {
|
||||
// R-1251: a restore that completed replaced the data a waiting put-back would have poured back — the household
|
||||
// chose this restore, so the old copy must never be poured over it when its drive returns. The copy stays.
|
||||
if _, err := m.settings.ClearRestoreHold(stackName); err != nil {
|
||||
m.logger.Printf("[ERROR] [backup] %s was restored, but its waiting put-back hold could not be cleared: %v", stackName, err)
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [backup] %s: restore completed — the waiting put-back (copy %s, set %s) is DROPPED; the copy stays on disk", stackName, h.SafetyDump, h.At)
|
||||
return
|
||||
}
|
||||
if !ok || h.Reason != settings.HoldReasonUpdateFailed {
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user