controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

controller/internal/web/logbuffer.go

@@ -1,6 +1,9 @@
 package web
 
 import (
+	"bufio"
+	"encoding/json"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -9,9 +12,9 @@ import (
 // LogEntry represents a single parsed log line.
 type LogEntry struct {
 	Timestamp time.Time `json:"timestamp"`
-	Level     string    `json:"level"`   // "DEBUG", "INFO", "WARN", "ERROR"
+	Level     string    `json:"level"` // "DEBUG", "INFO", "WARN", "ERROR"
 	Message   string    `json:"message"`
-	Source    string    `json:"source"`  // "file.go:123" if Lshortfile enabled
+	Source    string    `json:"source"` // "file.go:123" if Lshortfile enabled
 }
 
 // LogBuffer is a thread-safe ring buffer that captures log output.
@@ -44,6 +47,14 @@ func (lb *LogBuffer) Write(p []byte) (n int, err error) {
 
 	entry := parseLine(line)
 
+	// fix-6 (CAMPAIGN-3): TRACE lines (periodic-job routine success — the every-cycle scheduler +
+	// refreshStatusLocked noise) are DROPPED from the ring so the finite capacity holds SIGNAL, not
+	// "nothing happened, again". They still reach stdout if the configured level allows. Failures and
+	// state changes are never TRACE, so this never loses them.
+	if levelPriority(entry.Level) < levelPriority("DEBUG") {
+		return len(p), nil
+	}
+
 	lb.mu.Lock()
 	lb.entries[lb.pos] = entry
 	lb.pos = (lb.pos + 1) % lb.size
@@ -144,6 +155,101 @@ func (lb *LogBuffer) Lines(maxBytes int) []string {
 	return lines[keepFrom:]
 }
 
+// snapshot returns the held entries in chronological order (oldest→newest).
+func (lb *LogBuffer) snapshot() []LogEntry {
+	lb.mu.RLock()
+	defer lb.mu.RUnlock()
+	total := lb.size
+	start := 0
+	if !lb.full {
+		total = lb.pos
+	} else {
+		start = lb.pos
+	}
+	out := make([]LogEntry, 0, total)
+	for i := 0; i < total; i++ {
+		out = append(out, lb.entries[(start+i)%lb.size])
+	}
+	return out
+}
+
+// SpillTo persists the ring to path as JSON-lines (one entry per line), newest-last, via an atomic
+// tmp+rename so a crash mid-write never corrupts the on-disk ring (fix-6, CAMPAIGN-3). The path MUST
+// be on the SSD/state dir — NEVER a NAS/HDD path (a ring that dies with the NAS is worse than none).
+func (lb *LogBuffer) SpillTo(path string) error {
+	entries := lb.snapshot()
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(f)
+	enc := json.NewEncoder(w)
+	for _, e := range entries {
+		if err := enc.Encode(e); err != nil {
+			f.Close()
+			os.Remove(tmp)
+			return err
+		}
+	}
+	if err := w.Flush(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// LoadFrom seeds the ring from a spill file written by SpillTo, keeping the newest `size` entries so
+// the pre-restart window is present in the viewer immediately (fix-6). Corruption-safe: a truncated
+// or partially-written line simply fails to parse and is SKIPPED — the ring loads what is valid and
+// NEVER panics or errors fatally (a missing file is a clean no-op). Call once at startup, before the
+// logger writes anything.
+func (lb *LogBuffer) LoadFrom(path string) {
+	f, err := os.Open(path)
+	if err != nil {
+		return // no prior spill (fresh box / first boot) — clean no-op
+	}
+	defer f.Close()
+	var loaded []LogEntry
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) // tolerate long lines
+	for sc.Scan() {
+		var e LogEntry
+		if err := json.Unmarshal(sc.Bytes(), &e); err != nil {
+			continue // skip a corrupt/truncated line — never fatal
+		}
+		loaded = append(loaded, e)
+	}
+	if len(loaded) == 0 {
+		return
+	}
+	if len(loaded) > lb.size {
+		loaded = loaded[len(loaded)-lb.size:] // keep the newest `size`
+	}
+	lb.mu.Lock()
+	defer lb.mu.Unlock()
+	for i, e := range loaded {
+		lb.entries[i] = e
+	}
+	if len(loaded) == lb.size {
+		lb.pos = 0
+		lb.full = true
+	} else {
+		lb.pos = len(loaded)
+		lb.full = false
+	}
+}
+
 // parseLine parses a single log line into a LogEntry.
 func parseLine(line string) LogEntry {
 	entry := LogEntry{
@@ -195,6 +301,8 @@ func extractLevel(s string) (string, string) {
 	msg := strings.TrimSpace(s[end+1:])
 
 	switch tag {
+	case "TRACE":
+		return "TRACE", msg
 	case "DEBUG":
 		return "DEBUG", msg
 	case "INFO":
@@ -214,6 +322,8 @@ func extractLevel(s string) (string, string) {
 // levelPriority returns numeric priority for log levels.
 func levelPriority(level string) int {
 	switch strings.ToUpper(level) {
+	case "TRACE":
+		return -1 // below DEBUG — dropped from the ring (fix-6 periodic-noise policy)
 	case "DEBUG":
 		return 0
 	case "INFO":