felhom-controller/controller/internal/backup/appstop_putback_wait.go

package backup

import (
	"fmt"
	"os"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// ── R-1251: a pending put-back blocks every start (review round 2, HUNT-1 H1) ─────────────────────────────────────
//
// D22 pours a cut restore's copy back at start (restore_crash_putback.go). For a DRIVE-backed app the copy lives on
// the app's drive (restoreUndoDir under the drive's namespace), and at start the drive is often not bound yet. Until
// this file Recover read the missing folder as „gone", went on to the gated start (refused: no drive) and kept the
// marker; the drive then bound, the boot recreate started the app on the cut restore's data (it asked only „held?"),
// the household wrote, and the NEXT controller start found the copy and poured it over everything written since.
//
// Now:
//   - a copy that cannot be seen while its drive is not bound is WAITING, never gone. Recover writes a restore hold
//     (settings.HoldReasonPutBackPending, the copy's folder in SafetyDump) and starts nothing. The hold is the ONE
//     storage every start path already honours — the household's Start/Restart/Update, the boot recreate, the drive
//     return, the media restart, the boot sweep, the night chain, the update leg — so no path needs its own check;
//   - when the drive binds, ResumePutBack pours the copy back FIRST and lifts the hold; only then may the app start;
//   - a put-back that FAILS (at start or on resume) leaves the app HELD with the D20 hold (HoldReasonRestoreMixed, the
//     household's „stopped half-way" sentence) and the copy on disk — never stopped-and-unheld, which Start or the boot
//     recreate would run and a later start would pour an old copy over (H2). The marker then lets go of that app: the
//     hold owns it, exactly as holdAppAfterMixedRestore does in-process;
//   - Stop pressed during the restore no longer skips the put-back after a cut (H7): the data is put back, the app
//     stays stopped as the household asked.
//
// The hold — not the marker — is the durable record of a waiting put-back, because the marker has one slot and the
// next operation's Begin replaces it (R-1253).
//
// Pinned by r1251_putback_waits_test.go.

// AppStopHolds is the hold store the guard writes (settings.Settings in production).
type AppStopHolds interface {
	SetRestoreHold(settings.RestoreHold) error
	GetRestoreHold(stack string) (settings.RestoreHold, bool)
	ClearRestoreHold(stack string) (bool, error)
	ListRestoreHolds() []settings.RestoreHold
}

// SetHolds wires the hold store. INIT-ONLY, before Recover. nil = the pre-R-1251 behaviour (no hold is written).
func (g *AppStopGuard) SetHolds(h AppStopHolds) {
	if g != nil {
		g.holds = h
	}
}

// SetUndoReachable wires the question „is the drive this app's restore copy lives on bound right now?" (main.go:
// the app's HDD_PATH is a live mountpoint). nil = every folder counts as reachable, so a missing copy is „gone".
func (g *AppStopGuard) SetUndoReachable(f func(stack, dir string) bool) {
	if g != nil {
		g.undoReachable = f
	}
}

type undoCopyState int

const (
	undoPresent undoCopyState = iota
	undoGone                  // its drive is bound and the folder is not there
	undoWaiting               // its drive is not bound (or the folder cannot be read): it may come back
)

func (s undoCopyState) String() string {
	switch s {
	case undoPresent:
		return "present"
	case undoGone:
		return "gone"
	default:
		return "waiting"
	}
}

// copyState decides present / gone / waiting for one recorded copy.
func (g *AppStopGuard) copyState(stack, dir string) (undoCopyState, error) {
	_, err := os.Stat(dir)
	if err == nil {
		return undoPresent, nil
	}
	if !os.IsNotExist(err) {
		return undoWaiting, err // an unreadable folder is not proof that it is gone
	}
	if g.undoReachable != nil && !g.undoReachable(stack, dir) {
		return undoWaiting, err
	}
	return undoGone, err
}

// holdPutBackPending writes the „waiting for the drive" hold. Returns whether the app is now held by SOME hold.
func (g *AppStopGuard) holdPutBackPending(stack, dir string) bool {
	if g.holds == nil {
		g.logger.Printf("[ERROR] [appstop] %s: its restore copy %s waits for its drive, but no hold store is wired — nothing durable refuses a start (R-1251)", stack, dir)
		return false
	}
	if h, ok := g.holds.GetRestoreHold(stack); ok && h.Reason != settings.HoldReasonPutBackPending {
		// Another hold already refuses every start, and its sentence names the harder problem. The marker keeps the
		// copy's folder, so the next start puts it back once the drive is there.
		g.logger.Printf("[WARN] [appstop] %s: its restore copy %s waits for its drive; the app is already held (%q, set %s) — that hold stays", stack, dir, h.Reason, h.At)
		return true
	}
	h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending, SafetyDump: dir}
	if err := g.holds.SetRestoreHold(h); err != nil {
		g.logger.Printf("[ERROR] [appstop] %s: the waiting put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
		return false
	}
	g.logger.Printf("[WARN] [appstop] %s: the cut restore's copy %s is on a drive that is not bound — the app is HELD until the copy is put back (R-1251)", stack, dir)
	return true
}

// holdPutBackFailed writes the D20 hold for a put-back that could not be done (H2). Returns whether it was written.
func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool {
	if g.holds == nil {
		return false
	}
	h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed,
		ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir}
	if err := g.holds.SetRestoreHold(h); err != nil {
		g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err)
		return false
	}
	g.logger.Printf("[ERROR] [appstop] %s: the interrupted restore could NOT be put back (%v) — the app is HELD for support, the copy stays at %s (R-1251 H2)", stack, cause, dir)
	return true
}

// clearPutBackPending lifts this app's waiting-put-back hold, and only that kind.
func (g *AppStopGuard) clearPutBackPending(stack string) {
	if g.holds == nil {
		return
	}
	if h, ok := g.holds.GetRestoreHold(stack); !ok || h.Reason != settings.HoldReasonPutBackPending {
		return
	}
	if _, err := g.holds.ClearRestoreHold(stack); err != nil {
		g.logger.Printf("[ERROR] [appstop] %s: the put-back ran, but its hold could not be cleared: %v — the app stays held", stack, err)
	}
}

// dropFromMarker lets the active marker forget these apps (and their copies): a hold now owns them, or their put-back
// is done. A marker left with no app is cleared. A marker that belongs to another operation (it does not hold them)
// is not touched.
func (g *AppStopGuard) dropFromMarker(names ...string) {
	if g == nil || g.path == "" || len(names) == 0 {
		return
	}
	m, ok := g.read()
	if !ok || !m.Active {
		return
	}
	drop := map[string]bool{}
	for _, n := range names {
		drop[n] = true
	}
	var keep []string
	for _, s := range m.Stacks {
		if !drop[s] {
			keep = append(keep, s)
		}
	}
	if len(keep) == len(m.Stacks) {
		return
	}
	if len(keep) == 0 {
		if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) {
			g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v", g.path, err)
		}
		return
	}
	m.Stacks = keep
	for n := range drop {
		delete(m.Undo, n)
	}
	if err := g.write(m); err != nil {
		g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without %v: %v", names, err)
	}
}

// forgetUndo removes these apps' copy folders from the active marker (their put-back is done; the copy is removed),
// so a later start never tries to pour back a folder that no longer holds the app's data.
func (g *AppStopGuard) forgetUndo(names ...string) {
	if g == nil || g.path == "" || len(names) == 0 {
		return
	}
	m, ok := g.read()
	if !ok || !m.Active || len(m.Undo) == 0 {
		return
	}
	changed := false
	for _, n := range names {
		if _, ok := m.Undo[n]; ok {
			delete(m.Undo, n)
			changed = true
		}
	}
	if !changed {
		return
	}
	if err := g.write(m); err != nil {
		g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without the used copies of %v: %v", names, err)
	}
}

// resumeMu serialises ResumePutBack: the boot recreate and the drive-return sweep can reach the same app at once, and
// a copy must be poured back exactly once.
var resumeMu sync.Mutex

// ResumePutBack runs a put-back that was WAITING for its drive. It returns true when nothing is owed for stack any
// more — there was no waiting put-back, or it ran now — and the caller may go on to start the app (its other holds
// still apply). false: the app stays held (the drive is still not bound, or the put-back failed and the app is now
// held for support). Nil-safe.
func (g *AppStopGuard) ResumePutBack(stack string) bool {
	if g == nil || g.holds == nil {
		return true
	}
	resumeMu.Lock()
	defer resumeMu.Unlock()
	h, ok := g.holds.GetRestoreHold(stack)
	if !ok || h.Reason != settings.HoldReasonPutBackPending {
		return true
	}
	dir := h.SafetyDump
	if m, ok := g.read(); ok && m.Active && m.Undo[stack] != "" {
		dir = m.Undo[stack]
	}
	if dir == "" || g.putBacker == nil {
		g.logger.Printf("[ERROR] [appstop] %s: a put-back waits, but its copy is not named (or no put-back is wired) — the app stays HELD", stack)
		return false
	}
	state, serr := g.copyState(stack, dir)
	switch state {
	case undoWaiting:
		g.logger.Printf("[INFO] [appstop] %s: the restore copy %s is still not reachable (%v) — the app stays held until its drive is bound", stack, dir, serr)
		return false
	case undoGone:
		g.holdPutBackFailed(stack, dir, fmt.Errorf("the restore copy %s is gone while its drive is bound", dir))
		g.dropFromMarker(stack)
		return false
	}
	if err := g.putBacker.PutBackInterruptedRestore(stack, dir); err != nil {
		if g.holdPutBackFailed(stack, dir, err) {
			g.dropFromMarker(stack)
		}
		return false
	}
	g.clearPutBackPending(stack)
	g.dropFromMarker(stack)
	g.logger.Printf("[WARN] [appstop] %s: the drive is bound — the interrupted restore is put back from %s BEFORE the app starts; its data is as before the restore (R-1251)", stack, dir)
	return true
}

// ResumeWaitingPutBacks runs every waiting put-back whose drive is bound now, and starts each app afterwards through
// the guard's gated starter — unless the household stopped it. Called at start (after Recover) and when a drive
// returns. Returns the apps started.
func (g *AppStopGuard) ResumeWaitingPutBacks() []string {
	if g == nil || g.holds == nil {
		return nil
	}
	var started []string
	for _, h := range g.holds.ListRestoreHolds() {
		if h.Reason != settings.HoldReasonPutBackPending {
			continue
		}
		if !g.ResumePutBack(h.Stack) {
			continue
		}
		if ws, ok := g.starter.(interface{ WantsStopped(string) bool }); ok && ws.WantsStopped(h.Stack) {
			g.logger.Printf("[INFO] [appstop] %s: put back; NOT starting it — the household stopped it", h.Stack)
			continue
		}
		if g.starter == nil {
			continue
		}
		if err := g.starter.StartStack(h.Stack); err != nil {
			g.logger.Printf("[WARN] [appstop] %s: put back, but the start was not done: %v", h.Stack, err)
			continue
		}
		started = append(started, h.Stack)
	}
	return started
}