R-1251 (P1): a pending put-back blocks every start — a cut restore's copy on an unbound drive waits (a restore hold), is poured back when the drive binds before any start; a failed put-back holds the app (H2); Stop during a restore still puts back (H7). Commit only, unreleased.

2026-10-24 19:07:02 +02:00
parent 2449223f10
commit bb2809dc68
16 changed files with 847 additions and 24 deletions
@@ -0,0 +1,285 @@
+package backup
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// ── R-1251: a pending put-back blocks every start (review round 2, HUNT-1 H1) ─────────────────────────────────────
+//
+// D22 pours a cut restore's copy back at start (restore_crash_putback.go). For a DRIVE-backed app the copy lives on
+// the app's drive (restoreUndoDir under the drive's namespace), and at start the drive is often not bound yet. Until
+// this file Recover read the missing folder as „gone", went on to the gated start (refused: no drive) and kept the
+// marker; the drive then bound, the boot recreate started the app on the cut restore's data (it asked only „held?"),
+// the household wrote, and the NEXT controller start found the copy and poured it over everything written since.
+//
+// Now:
+//   - a copy that cannot be seen while its drive is not bound is WAITING, never gone. Recover writes a restore hold
+//     (settings.HoldReasonPutBackPending, the copy's folder in SafetyDump) and starts nothing. The hold is the ONE
+//     storage every start path already honours — the household's Start/Restart/Update, the boot recreate, the drive
+//     return, the media restart, the boot sweep, the night chain, the update leg — so no path needs its own check;
+//   - when the drive binds, ResumePutBack pours the copy back FIRST and lifts the hold; only then may the app start;
+//   - a put-back that FAILS (at start or on resume) leaves the app HELD with the D20 hold (HoldReasonRestoreMixed, the
+//     household's „stopped half-way" sentence) and the copy on disk — never stopped-and-unheld, which Start or the boot
+//     recreate would run and a later start would pour an old copy over (H2). The marker then lets go of that app: the
+//     hold owns it, exactly as holdAppAfterMixedRestore does in-process;
+//   - Stop pressed during the restore no longer skips the put-back after a cut (H7): the data is put back, the app
+//     stays stopped as the household asked.
+//
+// The hold — not the marker — is the durable record of a waiting put-back, because the marker has one slot and the
+// next operation's Begin replaces it (R-1253).
+//
+// Pinned by r1251_putback_waits_test.go.
+
+// AppStopHolds is the hold store the guard writes (settings.Settings in production).
+type AppStopHolds interface {
+	SetRestoreHold(settings.RestoreHold) error
+	GetRestoreHold(stack string) (settings.RestoreHold, bool)
+	ClearRestoreHold(stack string) (bool, error)
+	ListRestoreHolds() []settings.RestoreHold
+}
+
+// SetHolds wires the hold store. INIT-ONLY, before Recover. nil = the pre-R-1251 behaviour (no hold is written).
+func (g *AppStopGuard) SetHolds(h AppStopHolds) {
+	if g != nil {
+		g.holds = h
+	}
+}
+
+// SetUndoReachable wires the question „is the drive this app's restore copy lives on bound right now?" (main.go:
+// the app's HDD_PATH is a live mountpoint). nil = every folder counts as reachable, so a missing copy is „gone".
+func (g *AppStopGuard) SetUndoReachable(f func(stack, dir string) bool) {
+	if g != nil {
+		g.undoReachable = f
+	}
+}
+
+type undoCopyState int
+
+const (
+	undoPresent undoCopyState = iota
+	undoGone                  // its drive is bound and the folder is not there
+	undoWaiting               // its drive is not bound (or the folder cannot be read): it may come back
+)
+
+func (s undoCopyState) String() string {
+	switch s {
+	case undoPresent:
+		return "present"
+	case undoGone:
+		return "gone"
+	default:
+		return "waiting"
+	}
+}
+
+// copyState decides present / gone / waiting for one recorded copy.
+func (g *AppStopGuard) copyState(stack, dir string) (undoCopyState, error) {
+	_, err := os.Stat(dir)
+	if err == nil {
+		return undoPresent, nil
+	}
+	if !os.IsNotExist(err) {
+		return undoWaiting, err // an unreadable folder is not proof that it is gone
+	}
+	if g.undoReachable != nil && !g.undoReachable(stack, dir) {
+		return undoWaiting, err
+	}
+	return undoGone, err
+}
+
+// holdPutBackPending writes the „waiting for the drive" hold. Returns whether the app is now held by SOME hold.
+func (g *AppStopGuard) holdPutBackPending(stack, dir string) bool {
+	if g.holds == nil {
+		g.logger.Printf("[ERROR] [appstop] %s: its restore copy %s waits for its drive, but no hold store is wired — nothing durable refuses a start (R-1251)", stack, dir)
+		return false
+	}
+	if h, ok := g.holds.GetRestoreHold(stack); ok && h.Reason != settings.HoldReasonPutBackPending {
+		// Another hold already refuses every start, and its sentence names the harder problem. The marker keeps the
+		// copy's folder, so the next start puts it back once the drive is there.
+		g.logger.Printf("[WARN] [appstop] %s: its restore copy %s waits for its drive; the app is already held (%q, set %s) — that hold stays", stack, dir, h.Reason, h.At)
+		return true
+	}
+	h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending, SafetyDump: dir}
+	if err := g.holds.SetRestoreHold(h); err != nil {
+		g.logger.Printf("[ERROR] [appstop] %s: the waiting put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
+		return false
+	}
+	g.logger.Printf("[WARN] [appstop] %s: the cut restore's copy %s is on a drive that is not bound — the app is HELD until the copy is put back (R-1251)", stack, dir)
+	return true
+}
+
+// holdPutBackFailed writes the D20 hold for a put-back that could not be done (H2). Returns whether it was written.
+func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
+	if g.holds == nil {
+		return false
+	}
+	h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
+		ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
+	if err := g.holds.SetRestoreHold(h); err != nil {
+		g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
+		return false
+	}
+	g.logger.Printf("[ERROR] [appstop] %s: the interrupted restore could NOT be put back (%v) — the app is HELD for support, the copy stays at %s (R-1251 H2)", stack, cause, dir)
+	return true
+}
+
+// clearPutBackPending lifts this app's waiting-put-back hold, and only that kind.
+func (g *AppStopGuard) clearPutBackPending(stack string) {
+	if g.holds == nil {
+		return
+	}
+	if h, ok := g.holds.GetRestoreHold(stack); !ok || h.Reason != settings.HoldReasonPutBackPending {
+		return
+	}
+	if _, err := g.holds.ClearRestoreHold(stack); err != nil {
+		g.logger.Printf("[ERROR] [appstop] %s: the put-back ran, but its hold could not be cleared: %v — the app stays held", stack, err)
+	}
+}
+
+// dropFromMarker lets the active marker forget these apps (and their copies): a hold now owns them, or their put-back
+// is done. A marker left with no app is cleared. A marker that belongs to another operation (it does not hold them)
+// is not touched.
+func (g *AppStopGuard) dropFromMarker(names ...string) {
+	if g == nil || g.path == "" || len(names) == 0 {
+		return
+	}
+	m, ok := g.read()
+	if !ok || !m.Active {
+		return
+	}
+	drop := map[string]bool{}
+	for _, n := range names {
+		drop[n] = true
+	}
+	var keep []string
+	for _, s := range m.Stacks {
+		if !drop[s] {
+			keep = append(keep, s)
+		}
+	}
+	if len(keep) == len(m.Stacks) {
+		return
+	}
+	if len(keep) == 0 {
+		if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) {
+			g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v", g.path, err)
+		}
+		return
+	}
+	m.Stacks = keep
+	for n := range drop {
+		delete(m.Undo, n)
+	}
+	if err := g.write(m); err != nil {
+		g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without %v: %v", names, err)
+	}
+}
+
+// forgetUndo removes these apps' copy folders from the active marker (their put-back is done; the copy is removed),
+// so a later start never tries to pour back a folder that no longer holds the app's data.
+func (g *AppStopGuard) forgetUndo(names ...string) {
+	if g == nil || g.path == "" || len(names) == 0 {
+		return
+	}
+	m, ok := g.read()
+	if !ok || !m.Active || len(m.Undo) == 0 {
+		return
+	}
+	changed := false
+	for _, n := range names {
+		if _, ok := m.Undo[n]; ok {
+			delete(m.Undo, n)
+			changed = true
+		}
+	}
+	if !changed {
+		return
+	}
+	if err := g.write(m); err != nil {
+		g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without the used copies of %v: %v", names, err)
+	}
+}
+
+// resumeMu serialises ResumePutBack: the boot recreate and the drive-return sweep can reach the same app at once, and
+// a copy must be poured back exactly once.
+var resumeMu sync.Mutex
+
+// ResumePutBack runs a put-back that was WAITING for its drive. It returns true when nothing is owed for stack any
+// more — there was no waiting put-back, or it ran now — and the caller may go on to start the app (its other holds
+// still apply). false: the app stays held (the drive is still not bound, or the put-back failed and the app is now
+// held for support). Nil-safe.
+func (g *AppStopGuard) ResumePutBack(stack string) bool {
+	if g == nil || g.holds == nil {
+		return true
+	}
+	resumeMu.Lock()
+	defer resumeMu.Unlock()
+	h, ok := g.holds.GetRestoreHold(stack)
+	if !ok || h.Reason != settings.HoldReasonPutBackPending {
+		return true
+	}
+	dir := h.SafetyDump
+	if m, ok := g.read(); ok && m.Active && m.Undo[stack] != "" {
+		dir = m.Undo[stack]
+	}
+	if dir == "" || g.putBacker == nil {
+		g.logger.Printf("[ERROR] [appstop] %s: a put-back waits, but its copy is not named (or no put-back is wired) — the app stays HELD", stack)
+		return false
+	}
+	state, serr := g.copyState(stack, dir)
+	switch state {
+	case undoWaiting:
+		g.logger.Printf("[INFO] [appstop] %s: the restore copy %s is still not reachable (%v) — the app stays held until its drive is bound", stack, dir, serr)
+		return false
+	case undoGone:
+		g.holdPutBackFailed(stack, dir, fmt.Errorf("the restore copy %s is gone while its drive is bound", dir))
+		g.dropFromMarker(stack)
+		return false
+	}
+	if err := g.putBacker.PutBackInterruptedRestore(stack, dir); err != nil {
+		if g.holdPutBackFailed(stack, dir, err) {
+			g.dropFromMarker(stack)
+		}
+		return false
+	}
+	g.clearPutBackPending(stack)
+	g.dropFromMarker(stack)
+	g.logger.Printf("[WARN] [appstop] %s: the drive is bound — the interrupted restore is put back from %s BEFORE the app starts; its data is as before the restore (R-1251)", stack, dir)
+	return true
+}
+
+// ResumeWaitingPutBacks runs every waiting put-back whose drive is bound now, and starts each app afterwards through
+// the guard's gated starter — unless the household stopped it. Called at start (after Recover) and when a drive
+// returns. Returns the apps started.
+func (g *AppStopGuard) ResumeWaitingPutBacks() []string {
+	if g == nil || g.holds == nil {
+		return nil
+	}
+	var started []string
+	for _, h := range g.holds.ListRestoreHolds() {
+		if h.Reason != settings.HoldReasonPutBackPending {
+			continue
+		}
+		if !g.ResumePutBack(h.Stack) {
+			continue
+		}
+		if ws, ok := g.starter.(interface{ WantsStopped(string) bool }); ok && ws.WantsStopped(h.Stack) {
+			g.logger.Printf("[INFO] [appstop] %s: put back; NOT starting it — the household stopped it", h.Stack)
+			continue
+		}
+		if g.starter == nil {
+			continue
+		}
+		if err := g.starter.StartStack(h.Stack); err != nil {
+			g.logger.Printf("[WARN] [appstop] %s: put back, but the start was not done: %v", h.Stack, err)
+			continue
+		}
+		started = append(started, h.Stack)
+	}
+	return started
+}