controller: fix-3 dead-app alerting + fix-6 ring cap/spill/spam (WIP, pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

controller/cmd/controller/main.go

@@ -24,7 +24,6 @@ import (
 	"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/assets"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
-	"gitea.dooplex.hu/admin/felhom-controller/internal/offsiteapply"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
 	cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
@@ -35,6 +34,7 @@ import (
 	"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/monitor"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/offsiteapply"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/quiesce"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/recovery"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/report"
@@ -86,6 +86,11 @@ func main() {
 	}
 
 	logger, logBuffer := setupLogger(cfg)
+	// fix-6 (CAMPAIGN-3): load the pre-restart debug-ring window back immediately, so a controller
+	// restart / container recreation no longer wipes the exact evidence an operator needs. The spill
+	// lives on the SSD state dir (DataDir — holds settings.json, survives recreation), NEVER a NAS path.
+	ringSpillPath := filepath.Join(cfg.Paths.DataDir, "debug-ring.log")
+	logBuffer.LoadFrom(ringSpillPath)
 	// v0.116.0: the debug ring is the controller_log_tail source (report self-tail channel).
 	report.SetControllerLogSource(logBuffer.Lines)
 
@@ -390,6 +395,29 @@ func main() {
 		return nil
 	})
 
+	// fix-3 (CAMPAIGN-3): a deployed app that is not running must be LOUD, not silent (the campaign's
+	// CWA sat dead 4 h; F11 then produced 4 silently-dead NAS apps per reboot). Runs on its own short
+	// cadence (faster than the 5-min health cycle) so a dead app surfaces quickly. Logs nothing on the
+	// quiet path (no ring spam — fix-6 friendly); the dashboard banner is state-based (self-clears) and
+	// the hub event fires once per running→down transition (notifier tracks it). A boot grace skips the
+	// controller's own startup settle so apps that legitimately take 30–60 s to come up don't alert.
+	sched.Every("deadapp-check", 30*time.Second, func(ctx context.Context) error {
+		if time.Since(startTime) < deadAppBootGrace {
+			return nil // still inside the startup settle window
+		}
+		dead, states := scanDeployedAppRunStates(stackMgr)
+		alertMgr.SetDeadAppAlerts(dead)
+		notifier.NotifyAppStartFailures(states)
+		return nil
+	})
+
+	// fix-6 (CAMPAIGN-3): periodically spill the debug ring to the SSD state dir so a HARD crash loses
+	// at most ~60 s of the window (a clean shutdown spills too). ≤30s interval → auto-quiet (no
+	// per-cycle scheduler line). SpillTo is atomic (tmp+rename) so it can never corrupt the ring file.
+	sched.Every("ring-spill", 30*time.Second, func(ctx context.Context) error {
+		return logBuffer.SpillTo(ringSpillPath)
+	})
+
 	// --- Central hub pusher (declared early so backup closure can reference it) ---
 	var hubPusher *report.Pusher
 	if cfg.Hub.URL != "" && cfg.Hub.APIKey != "" {
@@ -936,6 +964,11 @@ func main() {
 	go func() {
 		sig := <-sigCh
 		logger.Printf("[INFO] Received signal %v, shutting down...", sig)
+		// fix-6: spill the debug ring on a clean shutdown so a `systemctl restart` / container recreate
+		// preserves the pre-restart window (the periodic spill already covers a hard crash to ≤60 s).
+		if err := logBuffer.SpillTo(ringSpillPath); err != nil {
+			logger.Printf("[WARN] debug-ring spill on shutdown failed: %v", err)
+		}
 		cancel()
 		if mailShim != nil {
 			mailShim.Close()
@@ -958,6 +991,30 @@ func main() {
 }
 
 // selfUpdateAuthMiddleware allows access via session auth (normal UI) OR hub API key bearer token (external).
+// deadAppBootGrace is the startup settle window before fix-3 evaluates deployed-app run states — apps
+// legitimately take 30–60 s to come up, so a shorter grace would false-alarm during the controller's
+// own boot. After the grace, an app that still isn't running alerts (the F11 dead-at-boot case).
+const deadAppBootGrace = 90 * time.Second
+
+// scanDeployedAppRunStates returns the fix-3 view of the deployed apps: the DEAD ones (for the
+// state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
+// transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks().
+func scanDeployedAppRunStates(mgr *stacks.Manager) ([]web.DeadApp, []notify.AppRunState) {
+	var dead []web.DeadApp
+	var states []notify.AppRunState
+	for _, st := range mgr.GetStacks() {
+		if !st.Deployed || st.Deploying {
+			continue
+		}
+		down := stacks.IsDownState(st.State)
+		states = append(states, notify.AppRunState{Name: st.Name, DisplayName: st.Meta.DisplayName, Down: down})
+		if down {
+			dead = append(dead, web.DeadApp{Name: st.Name, DisplayName: st.Meta.DisplayName, State: string(st.State)})
+		}
+	}
+	return dead, states
+}
+
 func selfUpdateAuthMiddleware(cfg *config.Config, webServer *web.Server, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Check bearer token first (for external API calls: hub, build scripts)
@@ -978,7 +1035,10 @@ func selfUpdateAuthMiddleware(cfg *config.Config, webServer *web.Server, next ht
 // exist without a config flip), while stdout (docker logs) keeps respecting
 // logging.level via the level filter. Lshortfile stays debug-only (unchanged).
 func setupLogger(cfg *config.Config) (*log.Logger, *web.LogBuffer) {
-	logBuffer := web.NewLogBuffer(1000)
+	// fix-6 (CAMPAIGN-3): 1000 wrapped in ~6.5 min under the campaign's ~2.6 entries/s load — the exact
+	// post-incident window an operator needs was the first thing lost. 5000 gives ≈32 min at that raw
+	// rate, and ≈50+ min now that the periodic-scheduler TRACE noise no longer enters the ring (2b).
+	logBuffer := web.NewLogBuffer(5000)
 	flags := log.LstdFlags
 	if cfg.Logging.Level == "debug" {
 		flags |= log.Lshortfile