R-1251 (P1): a pending put-back blocks every start — a cut restore's copy on an unbound drive waits (a restore hold), is poured back when the drive binds before any start; a failed put-back holds the app (H2); Stop during a restore still puts back (H7). Commit only, unreleased.

2026-10-24 19:07:02 +02:00
parent 2449223f10
commit bb2809dc68
16 changed files with 847 additions and 24 deletions
@@ -133,6 +133,10 @@ type AppStopGuard struct {
 	// putBacker (D22, R-1195) pours a restore's undo copy back before Recover restarts the app. nil = no put-back
 	// (the pre-D22 restart, unchanged).
 	putBacker InterruptedRestorePutBacker
+	// holds and undoReachable (R-1251, appstop_putback_wait.go): where a waiting or failed put-back is HELD, and
+	// whether the drive a copy lives on is bound. nil = the pre-R-1251 behaviour.
+	holds         AppStopHolds
+	undoReachable func(stack, dir string) bool

 	// suppressed (R-330) is the in-memory "the alarm must not fire for these, we are holding them"
 	// set. It is deliberately NOT persisted: after a restart the guard no longer holds anything —
@@ -162,6 +166,11 @@ type AppStopRecovery struct {
 	// PutBack are apps whose interrupted restore was undone from its copy before the restart (D22, R-1195): their
 	// data is as it was before the restore.
 	PutBack []string
+	// Waiting (R-1251) are apps whose restore copy is on a drive that is not bound: HELD, not started, put back first
+	// when the drive binds (ResumePutBack). Like Refused, the holder owns them and they do not alarm.
+	Waiting []string
+	// Held (R-1251 H2) are apps whose put-back FAILED and that are now held for support (the D20 hold). They alarm.
+	Held []string
 }

 // Alarming reports whether this recovery is worth paging an operator about. A recovery that only
@@ -171,7 +180,7 @@ func (r *AppStopRecovery) Alarming() bool {
 	if r == nil {
 		return false
 	}
-	return len(r.Failed) > 0 || len(r.Restarted) > 0
+	return len(r.Failed) > 0 || len(r.Held) > 0 || len(r.Restarted) > 0
 }

 // Message is the operator-facing headline for an interrupted operation.
@@ -179,14 +188,22 @@ func (r *AppStopRecovery) Message() string {
 	if r == nil {
 		return ""
 	}
-	if len(r.Failed) > 0 {
+	if len(r.Failed) > 0 || len(r.Held) > 0 {
+		bad := len(r.Failed) + len(r.Held)
 		m := fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted",
-			r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed))
+			r.Reason.humanReason(), bad, len(r.Restarted)+bad)
+		if len(r.Held) > 0 {
+			m += fmt.Sprintf(" (%d could not be put back and are HELD for support)", len(r.Held))
+		}
 		if len(r.Refused) > 0 {
 			m += fmt.Sprintf(" (a further %d are held by an absent drive and are not counted as failures)", len(r.Refused))
 		}
 		return m
 	}
+	if len(r.Waiting) > 0 && len(r.Restarted) == 0 {
+		return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are HELD until their data drive is bound: the copy taken before the change is on it and is put back first",
+			r.Reason.humanReason(), len(r.Waiting))
+	}
 	if len(r.Refused) > 0 && len(r.Restarted) == 0 {
 		return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are left stopped and HELD: their data drive is not available, so the drive gate restarts them when it returns",
 			r.Reason.humanReason(), len(r.Refused))
@@ -215,6 +232,12 @@ func (r *AppStopRecovery) Detail() string {
 	if len(r.PutBack) > 0 {
 		d += fmt.Sprintf(" put_back=%v", r.PutBack)
 	}
+	if len(r.Waiting) > 0 {
+		d += fmt.Sprintf(" put_back_waiting_for_drive=%v", r.Waiting)
+	}
+	if len(r.Held) > 0 {
+		d += fmt.Sprintf(" put_back_failed_held=%v", r.Held)
+	}
 	return d
 }

@@ -352,28 +375,48 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {

 	res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt}
 	hs, _ := g.starter.(interface{ WantsStopped(string) bool })
+	var usedUndo []string // apps whose copy was poured back: the marker must not name it again
 	for _, name := range m.Stacks {
-		if hs != nil && hs.WantsStopped(name) {
-			// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down.
-			// It is owed nothing; the marker still clears below.
-			g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
-			continue
-		}
+		// R-721 (v0.283.1): the household pressed Stop while the interrupted operation had the app down — it is owed
+		// no restart. R-1251 H7: it is still owed its put-back, so the stop is honoured AFTER the copy is poured back.
+		wantsStopped := hs != nil && hs.WantsStopped(name)
 		if dir := m.Undo[name]; dir != "" && g.putBacker != nil {
 			// D22 (R-1195): the restore was cut after its copy of the live data was taken — the app may be on the
-			// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery
-			// (the marker is kept, so the next startup tries again) and the app is not started on unknown data.
-			if _, statErr := os.Stat(dir); statErr != nil {
-				g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
-			} else if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
-				g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
-				res.Failed = append(res.Failed, name)
+			// backup's data, or on a mix. Pour the copy back first; a put-back that fails is a FAILED recovery and the
+			// app is not started on unknown data.
+			state, statErr := g.copyState(name, dir)
+			switch state {
+			case undoWaiting:
+				// R-1251: the copy is on a drive that is not bound yet. It is not „gone": the app is HELD (every start
+				// path honours the hold) and the copy is poured back when the drive binds (ResumePutBack).
+				g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s cannot be reached yet (%v) — NOT starting it; the put-back waits for its drive", name, dir, statErr)
+				g.holdPutBackPending(name, dir)
+				res.Waiting = append(res.Waiting, name)
 				continue
-			} else {
+			case undoGone:
+				g.logger.Printf("[WARN] [appstop] crash recovery: %s's restore copy %s is gone (%v) — restarting as before", name, dir, statErr)
+			default:
+				if err := g.putBacker.PutBackInterruptedRestore(name, dir); err != nil {
+					g.logger.Printf("[ERROR] [appstop] crash recovery: putting %s back from %s FAILED: %v — NOT starting it", name, dir, err)
+					// R-1251 H2: HELD (the D20 hold), never stopped-and-unheld. Without a hold store the old way stays:
+					// Failed, and the marker is kept so the next start tries again.
+					if g.holdPutBackFailed(name, dir, err) {
+						res.Held = append(res.Held, name)
+					} else {
+						res.Failed = append(res.Failed, name)
+					}
+					continue
+				}
 				g.logger.Printf("[WARN] [appstop] crash recovery: %s's interrupted restore is undone — its data is as before the restore (R-1195)", name)
+				g.clearPutBackPending(name)
 				res.PutBack = append(res.PutBack, name)
+				usedUndo = append(usedUndo, name)
 			}
 		}
+		if wantsStopped {
+			g.logger.Printf("[INFO] [appstop] crash recovery: NOT restarting %s — the household stopped it", name)
+			continue
+		}
 		if err := g.starter.StartStack(name); err != nil {
 			// R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be
 			// started (an absent data drive), so the app is left down deliberately and the holder
@@ -395,6 +438,8 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
 	sort.Strings(res.Refused)
 	sort.Strings(res.Restarted)
 	sort.Strings(res.PutBack)
+	sort.Strings(res.Waiting)
+	sort.Strings(res.Held)

 	// The marker is kept for BOTH unfinished outcomes, for the same reason and with different
 	// urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished
@@ -403,11 +448,16 @@ func (g *AppStopGuard) Recover() *AppStopRecovery {
 	if len(res.Failed) > 0 {
 		g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v",
 			len(res.Failed), res.Failed)
+		g.dropFromMarker(res.Held...)
+		g.forgetUndo(usedUndo...)
 		return res
 	}
-	if len(res.Refused) > 0 {
-		g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent) — KEEPING the marker; this is the gate working, not a fault: %v",
-			len(res.Refused), res.Refused)
+	if len(res.Refused) > 0 || len(res.Waiting) > 0 {
+		g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent: %v; put-back waiting for its drive: %v) — KEEPING the marker; this is the gate working, not a fault",
+			len(res.Refused)+len(res.Waiting), res.Refused, res.Waiting)
+		// The held (H2) apps and the used copies leave the marker; the rest of it stays for the next start.
+		g.dropFromMarker(res.Held...)
+		g.forgetUndo(usedUndo...)
 		return res
 	}
 	g.End()