controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-24 12:18:39 +02:00
parent 7c3b3a9694
commit 3c6b49b31c
141 changed files with 3401 additions and 237 deletions
@@ -0,0 +1,90 @@
+package main
+
+import (
+	"go/ast"
+	"io"
+	"log"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
+)
+
+// `09` §3 decision 28 (v0.269.0) — the box stops a crash loop / OOM storm; never inside a backup's,
+// a quiesce's or an update's own stop, never twice, never an app already held.
+func TestD28_StopHoldTellInOrder_AndTheSkips(t *testing.T) {
+	var calls []string
+	d := unhealthyDeps{
+		verdicts: func(time.Time, []stacks.OOMContainer) []stacks.UnhealthyVerdict {
+			return []stacks.UnhealthyVerdict{
+				{Stack: "gokapi", Kind: stacks.UnhealthyCrashLoop, Count: 7, Window: 10 * time.Minute},
+				{Stack: "quiesced", Kind: stacks.UnhealthyCrashLoop, Count: 7},
+				{Stack: "busy", Kind: stacks.UnhealthyCrashLoop, Count: 7},
+				{Stack: "held", Kind: stacks.UnhealthyOOMStorm, Count: 20},
+			}
+		},
+		suppressed: func() map[string]bool { return map[string]bool{"quiesced": true} },
+		busy:       func(n string) (bool, string) { return n == "busy", "a restore is running" },
+		holdKind: func(n string) string {
+			if n == "held" {
+				return "update_failed"
+			}
+			return ""
+		},
+		stop:   func(n string) error { calls = append(calls, "stop:"+n); return nil },
+		hold:   func(n, k string, _ time.Time) (int, error) { calls = append(calls, "hold:"+n+":"+k); return 1, nil },
+		notify: func(v stacks.UnhealthyVerdict, trip int) { calls = append(calls, "notify:"+v.Stack) },
+	}
+	stopped := stopUnhealthyApps(log.New(io.Discard, "", 0), d, nil, time.Now())
+	if strings.Join(calls, ",") != "stop:gokapi,hold:gokapi:crash_loop,notify:gokapi" {
+		t.Fatalf("calls = %v — want gokapi stopped, held, told; the quiesced, busy and held apps untouched", calls)
+	}
+	if len(stopped) != 1 || stopped[0] != "gokapi" {
+		t.Fatalf("stopped = %v", stopped)
+	}
+}
+
+// A failed stop is neither held nor announced (the household would read "we stopped it" over a running app).
+func TestD28_AFailedStopIsNotAnnounced(t *testing.T) {
+	var calls []string
+	d := unhealthyDeps{
+		verdicts: func(time.Time, []stacks.OOMContainer) []stacks.UnhealthyVerdict {
+			return []stacks.UnhealthyVerdict{{Stack: "x", Kind: "crash_loop"}}
+		},
+		suppressed: func() map[string]bool { return nil },
+		busy:       func(string) (bool, string) { return false, "" },
+		holdKind:   func(string) string { return "" },
+		stop:       func(string) error { return io.ErrUnexpectedEOF },
+		hold:       func(string, string, time.Time) (int, error) { calls = append(calls, "hold"); return 1, nil },
+		notify:     func(stacks.UnhealthyVerdict, int) { calls = append(calls, "notify") },
+	}
+	stopUnhealthyApps(log.New(io.Discard, "", 0), d, nil, time.Now())
+	if len(calls) != 0 {
+		t.Fatalf("calls = %v after a failed stop", calls)
+	}
+}
+
+// The wiring: main's dead-app loop calls stopUnhealthyApps with the detector.
+func TestD28_TheLoopStopsUnhealthyApps(t *testing.T) {
+	found, withDetector := false, false
+	ast.Inspect(mainBody(t), func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "stopUnhealthyApps" {
+			found = true
+			ast.Inspect(call, func(m ast.Node) bool {
+				if sel, ok := m.(*ast.SelectorExpr); ok && sel.Sel.Name == "ObserveUnhealthy" {
+					withDetector = true
+				}
+				return true
+			})
+		}
+		return true
+	})
+	if !found || !withDetector {
+		t.Fatalf("main calls stopUnhealthyApps=%v with ObserveUnhealthy=%v — decision 28 is built and never run", found, withDetector)
+	}
+}
@@ -864,7 +864,8 @@ func main() {
 		alertMgr.SetDeadAppAlerts(dead)
 		notifier.NotifyAppStartFailures(states)
 		// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
-		if ooms, oerr := stackMgr.ScanOOMKilled(); oerr != nil {
+		ooms, oerr := stackMgr.ScanOOMKilled()
+		if oerr != nil {
 			logger.Printf("[WARN] [deadapp] OOM scan failed: %v", oerr)
 		} else {
 			for _, o := range ooms {
@@ -872,6 +873,35 @@ func main() {
 				notifier.NotifyAppOOM(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
 			}
 		}
+		// v0.269.0 (`09` §3 decision 28, R-667): a crash loop or an OOM storm is STOPPED by the box.
+		stopUnhealthyApps(logger, unhealthyDeps{
+			verdicts: stackMgr.ObserveUnhealthy,
+			suppressed: func() map[string]bool {
+				return unionSuppressed(unionSuppressed(quiesceLoop.SuppressedStacks(), appStopGuard.SuppressedStacks()), stackMgr.UpdatingStacks())
+			},
+			busy: func(name string) (bool, string) {
+				if backupMgr == nil {
+					return false, ""
+				}
+				return backupMgr.UpdateBusy(name)
+			},
+			holdKind: func(name string) string {
+				if backupMgr == nil {
+					return ""
+				}
+				return backupMgr.HoldKind(name)
+			},
+			stop: stackMgr.StopStack,
+			hold: func(name, kind string, at time.Time) (int, error) {
+				if backupMgr == nil {
+					return 0, fmt.Errorf("backup is not enabled on this box — the stop cannot be held")
+				}
+				return backupMgr.HoldUnhealthy(name, kind, at)
+			},
+			notify: func(v stacks.UnhealthyVerdict, trip int) {
+				notifier.NotifyAppStoppedUnhealthy(v.Stack, v.Kind, v.Count, v.Window, trip)
+			},
+		}, ooms, time.Now())
 		deadAppScans++
 		noteDeadAppScan(logger, deadAppScans, len(states), len(dead))
 		noteUnknownIntentSuppressions(logger, states)
@@ -3586,6 +3616,14 @@ func (a *updateGuardsAdapter) HoldAfterFailedUpdate(name string, at time.Time, r
 	return err
 }

+// HoldKind names the hold in force (v0.269.0) — the page offers Start for an unhealthy stop.
+func (a *updateGuardsAdapter) HoldKind(name string) string {
+	if a.b == nil {
+		return ""
+	}
+	return a.b.HoldKind(name)
+}
+
 // HoldNoWholeCopy is the page's half of R-659: a hold naming no copy gets no restore button.
 func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
 	if a.b == nil {
@@ -3593,3 +3631,54 @@ func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
 	}
 	return a.b.HoldNoWholeCopy(name)
 }
+
+// ── Decision 28 (v0.269.0): the box stops an app in a crash loop or an out-of-memory storm ──────────
+
+// unhealthyDeps are the seams of stopUnhealthyApps — every one wired in main to the real component.
+type unhealthyDeps struct {
+	verdicts   func(now time.Time, ooms []stacks.OOMContainer) []stacks.UnhealthyVerdict
+	suppressed func() map[string]bool
+	busy       func(name string) (bool, string)
+	holdKind   func(name string) string
+	stop       func(name string) error
+	hold       func(name, kind string, at time.Time) (int, error)
+	notify     func(v stacks.UnhealthyVerdict, trip int)
+}
+
+// stopUnhealthyApps acts on the detector's verdicts. NEVER during a deploy or an update (the detector
+// skips those), a backup's or a restore's own stop, an app-data operation or a whole-guest quiesce (the
+// suppression sets, `08` §5), or on an app already held. Order: stop, then the hold (so no start path
+// revives it), then the events. Returns the apps stopped.
+func stopUnhealthyApps(logger *log.Logger, d unhealthyDeps, ooms []stacks.OOMContainer, now time.Time) []string {
+	var stopped []string
+	vs := d.verdicts(now, ooms)
+	if len(vs) == 0 {
+		return nil
+	}
+	sup := d.suppressed()
+	for _, v := range vs {
+		switch {
+		case sup[v.Stack]:
+			logger.Printf("[INFO] [deadapp] %s crossed the %s threshold (%d in %s) while a backup/quiesce/update holds it — NOT stopped", v.Stack, v.Kind, v.Count, v.Window)
+			continue
+		case d.holdKind(v.Stack) != "":
+			continue
+		}
+		if busy, why := d.busy(v.Stack); busy {
+			logger.Printf("[INFO] [deadapp] %s crossed the %s threshold but %s — NOT stopped", v.Stack, v.Kind, why)
+			continue
+		}
+		logger.Printf("[WARN] [deadapp] %s: %s — %d in %s; STOPPING it (decision 28)", v.Stack, v.Kind, v.Count, v.Window)
+		if err := d.stop(v.Stack); err != nil {
+			logger.Printf("[ERROR] [deadapp] stopping %s failed: %v — not held, not announced", v.Stack, err)
+			continue
+		}
+		trip, err := d.hold(v.Stack, v.Kind, now)
+		if err != nil {
+			logger.Printf("[ERROR] [deadapp] %s: %v", v.Stack, err)
+		}
+		d.notify(v, trip)
+		stopped = append(stopped, v.Stack)
+	}
+	return stopped
+}