controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/cmd/controller/main.go

@@ -864,7 +864,8 @@ func main() {
 		alertMgr.SetDeadAppAlerts(dead)
 		notifier.NotifyAppStartFailures(states)
 		// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
-		if ooms, oerr := stackMgr.ScanOOMKilled(); oerr != nil {
+		ooms, oerr := stackMgr.ScanOOMKilled()
+		if oerr != nil {
 			logger.Printf("[WARN] [deadapp] OOM scan failed: %v", oerr)
 		} else {
 			for _, o := range ooms {
@@ -872,6 +873,35 @@ func main() {
 				notifier.NotifyAppOOM(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
 			}
 		}
+		// v0.269.0 (`09` §3 decision 28, R-667): a crash loop or an OOM storm is STOPPED by the box.
+		stopUnhealthyApps(logger, unhealthyDeps{
+			verdicts: stackMgr.ObserveUnhealthy,
+			suppressed: func() map[string]bool {
+				return unionSuppressed(unionSuppressed(quiesceLoop.SuppressedStacks(), appStopGuard.SuppressedStacks()), stackMgr.UpdatingStacks())
+			},
+			busy: func(name string) (bool, string) {
+				if backupMgr == nil {
+					return false, ""
+				}
+				return backupMgr.UpdateBusy(name)
+			},
+			holdKind: func(name string) string {
+				if backupMgr == nil {
+					return ""
+				}
+				return backupMgr.HoldKind(name)
+			},
+			stop: stackMgr.StopStack,
+			hold: func(name, kind string, at time.Time) (int, error) {
+				if backupMgr == nil {
+					return 0, fmt.Errorf("backup is not enabled on this box — the stop cannot be held")
+				}
+				return backupMgr.HoldUnhealthy(name, kind, at)
+			},
+			notify: func(v stacks.UnhealthyVerdict, trip int) {
+				notifier.NotifyAppStoppedUnhealthy(v.Stack, v.Kind, v.Count, v.Window, trip)
+			},
+		}, ooms, time.Now())
 		deadAppScans++
 		noteDeadAppScan(logger, deadAppScans, len(states), len(dead))
 		noteUnknownIntentSuppressions(logger, states)
@@ -3586,6 +3616,14 @@ func (a *updateGuardsAdapter) HoldAfterFailedUpdate(name string, at time.Time, r
 	return err
 }
 
+// HoldKind names the hold in force (v0.269.0) — the page offers Start for an unhealthy stop.
+func (a *updateGuardsAdapter) HoldKind(name string) string {
+	if a.b == nil {
+		return ""
+	}
+	return a.b.HoldKind(name)
+}
+
 // HoldNoWholeCopy is the page's half of R-659: a hold naming no copy gets no restore button.
 func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
 	if a.b == nil {
@@ -3593,3 +3631,54 @@ func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
 	}
 	return a.b.HoldNoWholeCopy(name)
 }
+
+// ── Decision 28 (v0.269.0): the box stops an app in a crash loop or an out-of-memory storm ──────────
+
+// unhealthyDeps are the seams of stopUnhealthyApps — every one wired in main to the real component.
+type unhealthyDeps struct {
+	verdicts   func(now time.Time, ooms []stacks.OOMContainer) []stacks.UnhealthyVerdict
+	suppressed func() map[string]bool
+	busy       func(name string) (bool, string)
+	holdKind   func(name string) string
+	stop       func(name string) error
+	hold       func(name, kind string, at time.Time) (int, error)
+	notify     func(v stacks.UnhealthyVerdict, trip int)
+}
+
+// stopUnhealthyApps acts on the detector's verdicts. NEVER during a deploy or an update (the detector
+// skips those), a backup's or a restore's own stop, an app-data operation or a whole-guest quiesce (the
+// suppression sets, `08` §5), or on an app already held. Order: stop, then the hold (so no start path
+// revives it), then the events. Returns the apps stopped.
+func stopUnhealthyApps(logger *log.Logger, d unhealthyDeps, ooms []stacks.OOMContainer, now time.Time) []string {
+	var stopped []string
+	vs := d.verdicts(now, ooms)
+	if len(vs) == 0 {
+		return nil
+	}
+	sup := d.suppressed()
+	for _, v := range vs {
+		switch {
+		case sup[v.Stack]:
+			logger.Printf("[INFO] [deadapp] %s crossed the %s threshold (%d in %s) while a backup/quiesce/update holds it — NOT stopped", v.Stack, v.Kind, v.Count, v.Window)
+			continue
+		case d.holdKind(v.Stack) != "":
+			continue
+		}
+		if busy, why := d.busy(v.Stack); busy {
+			logger.Printf("[INFO] [deadapp] %s crossed the %s threshold but %s — NOT stopped", v.Stack, v.Kind, why)
+			continue
+		}
+		logger.Printf("[WARN] [deadapp] %s: %s — %d in %s; STOPPING it (decision 28)", v.Stack, v.Kind, v.Count, v.Window)
+		if err := d.stop(v.Stack); err != nil {
+			logger.Printf("[ERROR] [deadapp] stopping %s failed: %v — not held, not announced", v.Stack, err)
+			continue
+		}
+		trip, err := d.hold(v.Stack, v.Kind, now)
+		if err != nil {
+			logger.Printf("[ERROR] [deadapp] %s: %v", v.Stack, err)
+		}
+		d.notify(v, trip)
+		stopped = append(stopped, v.Stack)
+	}
+	return stopped
+}