controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

controller/internal/web/logbuffer_spill_test.go

@@ -0,0 +1,111 @@
+package web
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+var timeZeroVal time.Time
+
+func timeZero() time.Time { return timeZeroVal }
+func itoa(i int) string   { return strconv.Itoa(i) }
+
+// fix-6: a periodic-job routine line at [TRACE] is DROPPED from the ring (protecting the
+// post-incident window), while a [WARN]/[ERROR] failure on the same job IS kept. COMPANION red-proof:
+// demote too broadly (a failure at TRACE) → the failure vanishes from the ring → an operator loses it.
+func TestLogBuffer_TraceDropped_FailureKept(t *testing.T) {
+	lb := NewLogBuffer(100)
+	lb.Write([]byte("2026/07/12 09:00:00 [TRACE] [scheduler] job status-refresh: finished in 2ms (err=<nil>)\n"))
+	lb.Write([]byte("2026/07/12 09:00:01 [DEBUG] [stacks] refreshStatusLocked: stack \"radarr\" → state=running\n"))
+	lb.Write([]byte("2026/07/12 09:00:02 [WARN] [backup] volume dump failed for radarr\n"))
+
+	entries, _ := lb.Entries("DEBUG", 100, timeZero())
+	var kept []string
+	for _, e := range entries {
+		kept = append(kept, e.Level+" "+e.Message)
+	}
+	joined := strings.Join(kept, "|")
+	if strings.Contains(joined, "finished in 2ms") {
+		t.Errorf("a TRACE periodic line was kept in the ring (fix-6): %q", joined)
+	}
+	if !strings.Contains(joined, "volume dump failed") {
+		t.Errorf("a WARN failure line was DROPPED from the ring — must be kept: %q", joined)
+	}
+	if !strings.Contains(joined, "refreshStatusLocked") {
+		t.Errorf("a DEBUG line should still be captured (only TRACE is dropped): %q", joined)
+	}
+}
+
+// fix-6: the ring spills to disk and loads back the newest entries — a controller restart preserves
+// the pre-restart window. Round-trip through SpillTo → LoadFrom.
+func TestLogBuffer_SpillLoadRoundTrip(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	src := NewLogBuffer(100)
+	for _, m := range []string{"alpha", "beta", "gamma"} {
+		src.Write([]byte("2026/07/12 09:00:00 [INFO] [x] " + m + "\n"))
+	}
+	if err := src.SpillTo(path); err != nil {
+		t.Fatalf("spill: %v", err)
+	}
+
+	dst := NewLogBuffer(100)
+	dst.LoadFrom(path)
+	entries, _ := dst.Entries("INFO", 100, timeZero())
+	if len(entries) != 3 {
+		t.Fatalf("loaded %d entries, want 3", len(entries))
+	}
+	if entries[0].Message != "alpha" || entries[2].Message != "gamma" {
+		t.Errorf("round-trip order/content wrong: %+v", entries)
+	}
+}
+
+// Load keeps only the NEWEST `size` entries when the spill holds more than the ring cap.
+func TestLogBuffer_LoadKeepsNewest(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	src := NewLogBuffer(1000)
+	for i := 0; i < 20; i++ {
+		src.Write([]byte("2026/07/12 09:00:00 [INFO] [x] msg" + itoa(i) + "\n"))
+	}
+	if err := src.SpillTo(path); err != nil {
+		t.Fatal(err)
+	}
+	small := NewLogBuffer(5) // smaller ring
+	small.LoadFrom(path)
+	entries, total := small.Entries("INFO", 100, timeZero())
+	if total != 5 || len(entries) != 5 {
+		t.Fatalf("small ring should hold the newest 5, got total=%d len=%d", total, len(entries))
+	}
+	if entries[len(entries)-1].Message != "msg19" {
+		t.Errorf("newest entry should be msg19, got %q", entries[len(entries)-1].Message)
+	}
+}
+
+// A corrupt/truncated spill file loads what is valid and NEVER panics (crash-during-spill safety).
+func TestLogBuffer_LoadCorruptFileSafe(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	// One valid JSON line, then a truncated/garbage line (a crash mid-write).
+	content := `{"timestamp":"2026-07-12T09:00:00Z","level":"INFO","message":"valid","source":""}` + "\n" +
+		`{"timestamp":"2026-07-12T09:00:01Z","level":"WARN","messa` // truncated
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+	lb := NewLogBuffer(100)
+	lb.LoadFrom(path) // must not panic
+	entries, _ := lb.Entries("INFO", 100, timeZero())
+	if len(entries) != 1 || entries[0].Message != "valid" {
+		t.Fatalf("corrupt spill should load the 1 valid entry, got %+v", entries)
+	}
+}
+
+// A missing spill file is a clean no-op.
+func TestLogBuffer_LoadMissingFileNoOp(t *testing.T) {
+	lb := NewLogBuffer(100)
+	lb.LoadFrom(filepath.Join(t.TempDir(), "does-not-exist.log")) // must not panic
+	if entries, total := lb.Entries("DEBUG", 100, timeZero()); total != 0 || len(entries) != 0 {
+		t.Fatalf("missing spill must leave an empty ring, got total=%d", total)
+	}
+}