controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

controller/internal/web/alerts.go

@@ -37,6 +37,11 @@ type AlertManager struct {
 	// is health-report-driven). nil = channel up. It is prepended in GetAlerts so a dead controller→
 	// agent link (disk/storage UI broken) is always visible regardless of the health-report cycle.
 	agentChannelAlert *Alert
+	// deadAppAlerts (fix-3, CAMPAIGN-3) is set each cycle by the health loop from the deployed-app
+	// running-state view (which is NOT in the health report — it comes from the stack manager). Same
+	// out-of-band, state-based, self-clearing model as agentChannelAlert: passing an empty slice when
+	// every deployed app is running clears the banner with no manual dismissal.
+	deadAppAlerts []Alert
 }
 
 // NewAlertManager creates a new AlertManager.
@@ -72,6 +77,63 @@ func (am *AlertManager) SetAgentChannelAlert(down bool, msg string) {
 	}
 }
 
+// DeadApp is a deployed app the health loop found not-running (fix-3). State is the container-state
+// string for the display (e.g. "stopped"/"exited").
+type DeadApp struct {
+	Name        string
+	DisplayName string
+	State       string
+}
+
+// deadAppGroupThreshold: above this many dead apps, collapse to ONE grouped alert (a reboot storm
+// with many NAS apps down should not paper the dashboard with a wall of banners — fix-3).
+const deadAppGroupThreshold = 3
+
+// buildDeadAppAlerts turns the dead-app list into dashboard alerts (WARN). ≤ threshold → one per app;
+// more → a single grouped alert. Pure → unit-tested. Empty list → nil (clears the banner).
+func buildDeadAppAlerts(dead []DeadApp) []Alert {
+	if len(dead) == 0 {
+		return nil
+	}
+	if len(dead) > deadAppGroupThreshold {
+		return []Alert{{
+			ID:       "deadapp-group",
+			Level:    "warning",
+			Message:  fmt.Sprintf("%d telepített alkalmazás nem fut — nézze meg a rendszermonitort", len(dead)),
+			Link:     "/monitoring",
+			LinkText: "Rendszermonitor",
+		}}
+	}
+	alerts := make([]Alert, 0, len(dead))
+	for _, d := range dead {
+		name := d.DisplayName
+		if name == "" {
+			name = d.Name
+		}
+		msg := "Telepített alkalmazás nem fut: " + name
+		if d.State != "" {
+			msg += " (" + d.State + ")"
+		}
+		alerts = append(alerts, Alert{
+			ID:       "deadapp-" + simpleHash(d.Name),
+			Level:    "warning",
+			Message:  msg,
+			Link:     "/monitoring",
+			LinkText: "Rendszermonitor",
+		})
+	}
+	return alerts
+}
+
+// SetDeadAppAlerts stores the current deployed-not-running banner set (fix-3). State-based: called
+// each health cycle with the live dead-app list (empty clears it). Included in GetAlerts.
+func (am *AlertManager) SetDeadAppAlerts(dead []DeadApp) {
+	alerts := buildDeadAppAlerts(dead)
+	am.mu.Lock()
+	am.deadAppAlerts = alerts
+	am.mu.Unlock()
+}
+
 // Refresh regenerates alerts from the latest health check report and config state.
 // Called after each health check cycle (every 5 minutes) and on storage state changes.
 func (am *AlertManager) Refresh(report *monitor.HealthReport, cfg *config.Config, backupMgr *backup.Manager, updateAvailable bool, latestVersion string, storagePaths ...[]settings.StoragePath) {
@@ -182,7 +244,7 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
 	am.mu.RLock()
 	defer am.mu.RUnlock()
 
-	if len(am.alerts) == 0 && am.agentChannelAlert == nil {
+	if len(am.alerts) == 0 && am.agentChannelAlert == nil && len(am.deadAppAlerts) == 0 {
 		return nil
 	}
 
@@ -196,6 +258,13 @@ func (am *AlertManager) GetAlerts(excludeIDs ...string) []Alert {
 	if am.agentChannelAlert != nil && !exclude[am.agentChannelAlert.ID] {
 		result = append(result, *am.agentChannelAlert)
 	}
+	// Dead-app banners (fix-3) — prepended after the channel alert: a deployed app being down is a
+	// high-signal state the operator/customer must see immediately.
+	for _, a := range am.deadAppAlerts {
+		if !exclude[a.ID] {
+			result = append(result, a)
+		}
+	}
 	for _, a := range am.alerts {
 		if exclude[a.ID] {
 			continue

controller/internal/web/deadapp_alert_test.go

@@ -0,0 +1,44 @@
+package web
+
+import (
+	"log"
+	"strings"
+	"testing"
+)
+
+// fix-3: a deployed-not-running app produces a self-clearing WARN dashboard banner; an empty list
+// clears it. COMPANION red-proof: skip SetDeadAppAlerts entirely (the pre-fix-3 silence) → GetAlerts
+// has no dead-app banner → the "not fut" assertion fails.
+func TestDeadAppAlerts_PresentAndSelfClearing(t *testing.T) {
+	am := NewAlertManager(log.New(nil, "", 0))
+
+	am.SetDeadAppAlerts([]DeadApp{{Name: "cwa", DisplayName: "Calibre-Web", State: "stopped"}})
+	got := am.GetAlerts()
+	if len(got) != 1 || got[0].Level != "warning" || !strings.Contains(got[0].Message, "Telepített alkalmazás nem fut: Calibre-Web") {
+		t.Fatalf("expected a WARN dead-app banner, got %+v", got)
+	}
+	if !strings.Contains(got[0].Message, "stopped") {
+		t.Errorf("banner should carry the state: %q", got[0].Message)
+	}
+
+	// The app recovers → an empty set clears the banner (state-based, no manual dismissal).
+	am.SetDeadAppAlerts(nil)
+	if got := am.GetAlerts(); len(got) != 0 {
+		t.Fatalf("dead-app banner must self-clear when the app recovers, got %+v", got)
+	}
+}
+
+// Above the group threshold, many dead apps collapse to ONE grouped banner (a reboot storm must not
+// paper the dashboard).
+func TestDeadAppAlerts_GroupedAboveThreshold(t *testing.T) {
+	am := NewAlertManager(log.New(nil, "", 0))
+	var many []DeadApp
+	for _, n := range []string{"radarr", "jellyfin", "navidrome", "calibre", "seerr"} {
+		many = append(many, DeadApp{Name: n, DisplayName: n, State: "stopped"})
+	}
+	am.SetDeadAppAlerts(many)
+	got := am.GetAlerts()
+	if len(got) != 1 || !strings.Contains(got[0].Message, "5 telepített alkalmazás nem fut") {
+		t.Fatalf("expected one grouped banner for %d dead apps, got %+v", len(many), got)
+	}
+}

controller/internal/web/logbuffer.go

@@ -1,6 +1,9 @@
 package web
 
 import (
+	"bufio"
+	"encoding/json"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -9,9 +12,9 @@ import (
 // LogEntry represents a single parsed log line.
 type LogEntry struct {
 	Timestamp time.Time `json:"timestamp"`
-	Level     string    `json:"level"`   // "DEBUG", "INFO", "WARN", "ERROR"
+	Level     string    `json:"level"` // "DEBUG", "INFO", "WARN", "ERROR"
 	Message   string    `json:"message"`
-	Source    string    `json:"source"`  // "file.go:123" if Lshortfile enabled
+	Source    string    `json:"source"` // "file.go:123" if Lshortfile enabled
 }
 
 // LogBuffer is a thread-safe ring buffer that captures log output.
@@ -44,6 +47,14 @@ func (lb *LogBuffer) Write(p []byte) (n int, err error) {
 
 	entry := parseLine(line)
 
+	// fix-6 (CAMPAIGN-3): TRACE lines (periodic-job routine success — the every-cycle scheduler +
+	// refreshStatusLocked noise) are DROPPED from the ring so the finite capacity holds SIGNAL, not
+	// "nothing happened, again". They still reach stdout if the configured level allows. Failures and
+	// state changes are never TRACE, so this never loses them.
+	if levelPriority(entry.Level) < levelPriority("DEBUG") {
+		return len(p), nil
+	}
+
 	lb.mu.Lock()
 	lb.entries[lb.pos] = entry
 	lb.pos = (lb.pos + 1) % lb.size
@@ -144,6 +155,101 @@ func (lb *LogBuffer) Lines(maxBytes int) []string {
 	return lines[keepFrom:]
 }
 
+// snapshot returns the held entries in chronological order (oldest→newest).
+func (lb *LogBuffer) snapshot() []LogEntry {
+	lb.mu.RLock()
+	defer lb.mu.RUnlock()
+	total := lb.size
+	start := 0
+	if !lb.full {
+		total = lb.pos
+	} else {
+		start = lb.pos
+	}
+	out := make([]LogEntry, 0, total)
+	for i := 0; i < total; i++ {
+		out = append(out, lb.entries[(start+i)%lb.size])
+	}
+	return out
+}
+
+// SpillTo persists the ring to path as JSON-lines (one entry per line), newest-last, via an atomic
+// tmp+rename so a crash mid-write never corrupts the on-disk ring (fix-6, CAMPAIGN-3). The path MUST
+// be on the SSD/state dir — NEVER a NAS/HDD path (a ring that dies with the NAS is worse than none).
+func (lb *LogBuffer) SpillTo(path string) error {
+	entries := lb.snapshot()
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(f)
+	enc := json.NewEncoder(w)
+	for _, e := range entries {
+		if err := enc.Encode(e); err != nil {
+			f.Close()
+			os.Remove(tmp)
+			return err
+		}
+	}
+	if err := w.Flush(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// LoadFrom seeds the ring from a spill file written by SpillTo, keeping the newest `size` entries so
+// the pre-restart window is present in the viewer immediately (fix-6). Corruption-safe: a truncated
+// or partially-written line simply fails to parse and is SKIPPED — the ring loads what is valid and
+// NEVER panics or errors fatally (a missing file is a clean no-op). Call once at startup, before the
+// logger writes anything.
+func (lb *LogBuffer) LoadFrom(path string) {
+	f, err := os.Open(path)
+	if err != nil {
+		return // no prior spill (fresh box / first boot) — clean no-op
+	}
+	defer f.Close()
+	var loaded []LogEntry
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) // tolerate long lines
+	for sc.Scan() {
+		var e LogEntry
+		if err := json.Unmarshal(sc.Bytes(), &e); err != nil {
+			continue // skip a corrupt/truncated line — never fatal
+		}
+		loaded = append(loaded, e)
+	}
+	if len(loaded) == 0 {
+		return
+	}
+	if len(loaded) > lb.size {
+		loaded = loaded[len(loaded)-lb.size:] // keep the newest `size`
+	}
+	lb.mu.Lock()
+	defer lb.mu.Unlock()
+	for i, e := range loaded {
+		lb.entries[i] = e
+	}
+	if len(loaded) == lb.size {
+		lb.pos = 0
+		lb.full = true
+	} else {
+		lb.pos = len(loaded)
+		lb.full = false
+	}
+}
+
 // parseLine parses a single log line into a LogEntry.
 func parseLine(line string) LogEntry {
 	entry := LogEntry{
@@ -195,6 +301,8 @@ func extractLevel(s string) (string, string) {
 	msg := strings.TrimSpace(s[end+1:])
 
 	switch tag {
+	case "TRACE":
+		return "TRACE", msg
 	case "DEBUG":
 		return "DEBUG", msg
 	case "INFO":
@@ -214,6 +322,8 @@ func extractLevel(s string) (string, string) {
 // levelPriority returns numeric priority for log levels.
 func levelPriority(level string) int {
 	switch strings.ToUpper(level) {
+	case "TRACE":
+		return -1 // below DEBUG — dropped from the ring (fix-6 periodic-noise policy)
 	case "DEBUG":
 		return 0
 	case "INFO":

controller/internal/web/logbuffer_spill_test.go

@@ -0,0 +1,111 @@
+package web
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+var timeZeroVal time.Time
+
+func timeZero() time.Time { return timeZeroVal }
+func itoa(i int) string   { return strconv.Itoa(i) }
+
+// fix-6: a periodic-job routine line at [TRACE] is DROPPED from the ring (protecting the
+// post-incident window), while a [WARN]/[ERROR] failure on the same job IS kept. COMPANION red-proof:
+// demote too broadly (a failure at TRACE) → the failure vanishes from the ring → an operator loses it.
+func TestLogBuffer_TraceDropped_FailureKept(t *testing.T) {
+	lb := NewLogBuffer(100)
+	lb.Write([]byte("2026/07/12 09:00:00 [TRACE] [scheduler] job status-refresh: finished in 2ms (err=<nil>)\n"))
+	lb.Write([]byte("2026/07/12 09:00:01 [DEBUG] [stacks] refreshStatusLocked: stack \"radarr\" → state=running\n"))
+	lb.Write([]byte("2026/07/12 09:00:02 [WARN] [backup] volume dump failed for radarr\n"))
+
+	entries, _ := lb.Entries("DEBUG", 100, timeZero())
+	var kept []string
+	for _, e := range entries {
+		kept = append(kept, e.Level+" "+e.Message)
+	}
+	joined := strings.Join(kept, "|")
+	if strings.Contains(joined, "finished in 2ms") {
+		t.Errorf("a TRACE periodic line was kept in the ring (fix-6): %q", joined)
+	}
+	if !strings.Contains(joined, "volume dump failed") {
+		t.Errorf("a WARN failure line was DROPPED from the ring — must be kept: %q", joined)
+	}
+	if !strings.Contains(joined, "refreshStatusLocked") {
+		t.Errorf("a DEBUG line should still be captured (only TRACE is dropped): %q", joined)
+	}
+}
+
+// fix-6: the ring spills to disk and loads back the newest entries — a controller restart preserves
+// the pre-restart window. Round-trip through SpillTo → LoadFrom.
+func TestLogBuffer_SpillLoadRoundTrip(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	src := NewLogBuffer(100)
+	for _, m := range []string{"alpha", "beta", "gamma"} {
+		src.Write([]byte("2026/07/12 09:00:00 [INFO] [x] " + m + "\n"))
+	}
+	if err := src.SpillTo(path); err != nil {
+		t.Fatalf("spill: %v", err)
+	}
+
+	dst := NewLogBuffer(100)
+	dst.LoadFrom(path)
+	entries, _ := dst.Entries("INFO", 100, timeZero())
+	if len(entries) != 3 {
+		t.Fatalf("loaded %d entries, want 3", len(entries))
+	}
+	if entries[0].Message != "alpha" || entries[2].Message != "gamma" {
+		t.Errorf("round-trip order/content wrong: %+v", entries)
+	}
+}
+
+// Load keeps only the NEWEST `size` entries when the spill holds more than the ring cap.
+func TestLogBuffer_LoadKeepsNewest(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	src := NewLogBuffer(1000)
+	for i := 0; i < 20; i++ {
+		src.Write([]byte("2026/07/12 09:00:00 [INFO] [x] msg" + itoa(i) + "\n"))
+	}
+	if err := src.SpillTo(path); err != nil {
+		t.Fatal(err)
+	}
+	small := NewLogBuffer(5) // smaller ring
+	small.LoadFrom(path)
+	entries, total := small.Entries("INFO", 100, timeZero())
+	if total != 5 || len(entries) != 5 {
+		t.Fatalf("small ring should hold the newest 5, got total=%d len=%d", total, len(entries))
+	}
+	if entries[len(entries)-1].Message != "msg19" {
+		t.Errorf("newest entry should be msg19, got %q", entries[len(entries)-1].Message)
+	}
+}
+
+// A corrupt/truncated spill file loads what is valid and NEVER panics (crash-during-spill safety).
+func TestLogBuffer_LoadCorruptFileSafe(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "debug-ring.log")
+	// One valid JSON line, then a truncated/garbage line (a crash mid-write).
+	content := `{"timestamp":"2026-07-12T09:00:00Z","level":"INFO","message":"valid","source":""}` + "\n" +
+		`{"timestamp":"2026-07-12T09:00:01Z","level":"WARN","messa` // truncated
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+	lb := NewLogBuffer(100)
+	lb.LoadFrom(path) // must not panic
+	entries, _ := lb.Entries("INFO", 100, timeZero())
+	if len(entries) != 1 || entries[0].Message != "valid" {
+		t.Fatalf("corrupt spill should load the 1 valid entry, got %+v", entries)
+	}
+}
+
+// A missing spill file is a clean no-op.
+func TestLogBuffer_LoadMissingFileNoOp(t *testing.T) {
+	lb := NewLogBuffer(100)
+	lb.LoadFrom(filepath.Join(t.TempDir(), "does-not-exist.log")) // must not panic
+	if entries, total := lb.Entries("DEBUG", 100, timeZero()); total != 0 || len(entries) != 0 {
+		t.Fatalf("missing spill must leave an empty ring, got total=%d", total)
+	}
+}