controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-24 12:18:39 +02:00
parent 7c3b3a9694
commit 3c6b49b31c
141 changed files with 3401 additions and 237 deletions
@@ -0,0 +1,102 @@
+package stacks
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"testing"
+	"time"
+)
+
+var uT0 = time.Date(2026, 9, 24, 9, 22, 0, 0, time.UTC)
+
+// gokapi's measured shape: Docker's back-off caps a steady crash loop at ~1 restart a minute (539 → 546
+// in 7 min on 9202). Sampled every 30 s, as the dead-app loop does.
+func gokapiSamples(minutes int) []unhealthySample {
+	var ss []unhealthySample
+	for i := 0; i <= minutes*2; i++ {
+		ss = append(ss, unhealthySample{at: uT0.Add(time.Duration(i) * 30 * time.Second), restarts: 539 + int64(i/2), kills: 0})
+	}
+	return ss
+}
+
+// TestR667_TheMeasuredCrashLoopTrips — the shape the old detector never saw.
+//
+// COMPANION RED-PROOF (REPORT): make judgeUnhealthy compare only the LAST TWO samples (the "clock that
+// resets at every look" shape of RestartingSince) — this test then fails at "gokapi's loop did not trip".
+func TestR667_TheMeasuredCrashLoopTrips(t *testing.T) {
+	if k, _, _ := judgeUnhealthy(gokapiSamples(5)); k != "" {
+		t.Fatalf("5 minutes of 1/min restarts tripped already (%s) — the threshold is 6 in 10 min", k)
+	}
+	k, n, w := judgeUnhealthy(gokapiSamples(7))
+	if k != UnhealthyCrashLoop || n < CrashLoopRestarts || w != CrashLoopWindow {
+		t.Fatalf("gokapi's loop did not trip: kind=%q n=%d", k, n)
+	}
+}
+
+func TestR667_SlowStartsAndResetsDoNotTrip(t *testing.T) {
+	// the worst healthy container measured on the demo boxes: one restart at first start
+	one := []unhealthySample{{at: uT0, restarts: 0}, {at: uT0.Add(30 * time.Second), restarts: 1}, {at: uT0.Add(9 * time.Minute), restarts: 1}}
+	if k, _, _ := judgeUnhealthy(one); k != "" {
+		t.Fatalf("a single first-start restart tripped: %s", k)
+	}
+	// compose recreated the container: RestartCount 40 → 0 → 3 is a reset, not -37 or +43
+	reset := []unhealthySample{{at: uT0, restarts: 40}, {at: uT0.Add(time.Minute), restarts: 0}, {at: uT0.Add(2 * time.Minute), restarts: 3}}
+	if k, _, _ := judgeUnhealthy(reset); k != "" {
+		t.Fatalf("a recreate read as a crash loop: %s", k)
+	}
+	// restarts spread over more than the window: 6 restarts in 30 min is not a loop
+	var spread []unhealthySample
+	for i := 0; i <= 6; i++ {
+		spread = append(spread, unhealthySample{at: uT0.Add(time.Duration(i) * 5 * time.Minute), restarts: int64(i)})
+	}
+	if k, _, _ := judgeUnhealthy(spread); k != "" {
+		t.Fatalf("6 restarts in 30 min tripped: %s", k)
+	}
+}
+
+func TestR667_OOMStormTrips(t *testing.T) {
+	ss := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(10 * time.Minute), kills: 12}, {at: uT0.Add(25 * time.Minute), kills: 23}}
+	if k, n, _ := judgeUnhealthy(ss); k != UnhealthyOOMStorm || n != 20 {
+		t.Fatalf("kind=%q n=%d, want an OOM storm of 20", k, n)
+	}
+	calm := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(25 * time.Minute), kills: 10}}
+	if k, _, _ := judgeUnhealthy(calm); k != "" {
+		t.Fatalf("7 kills tripped: %s", k)
+	}
+}
+
+// ObserveUnhealthy through its docker seam: RestartCount summed per app, one verdict per episode, and an
+// app that is updating is never judged.
+func TestR667_ObserveSumsPerAppAndSkipsUpdating(t *testing.T) {
+	m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
+		"gokapi": {Name: "gokapi", Deployed: true, Containers: []ContainerInfo{{Name: "gokapi"}}},
+		"romm":   {Name: "romm", Deployed: true, Updating: true, Containers: []ContainerInfo{{Name: "romm"}}},
+	}}
+	count := int64(539)
+	m.execFn = func(name string, args ...string) (string, error) {
+		a := strings.Join(args, " ")
+		out := ""
+		if strings.Contains(a, "gokapi") {
+			out += fmt.Sprintf("/gokapi|%d\n", count)
+		}
+		if strings.Contains(a, "romm") {
+			out += "/romm|999\n"
+		}
+		return out, nil
+	}
+	var got []UnhealthyVerdict
+	for i := 0; i <= 16 && len(got) == 0; i++ {
+		got = m.ObserveUnhealthy(uT0.Add(time.Duration(i)*30*time.Second), nil)
+		if i%2 == 1 {
+			count++
+		}
+	}
+	if len(got) != 1 || got[0].Stack != "gokapi" || got[0].Kind != UnhealthyCrashLoop {
+		t.Fatalf("verdicts = %+v, want exactly gokapi's crash loop", got)
+	}
+	if again := m.ObserveUnhealthy(uT0.Add(9*time.Minute), nil); len(again) != 0 {
+		t.Fatalf("the same episode tripped twice: %+v", again)
+	}
+}