R-856: after a crash boot of the host, app mails wait ~15 minutes; a normal boot keeps 90 s (09 decision 143)

The dead-app check (source of app_start_failed and app_stopped_unhealthy) now gates on a crash-aware
boot grace (internal/crashboot): 15 min when the host crash guard's last boot was UNCLEAN and within
30 min of the controller start, otherwise 90 s. The fact is read from the agent's local API
(GET /host/crash-guard, agentapi.Client.CrashGuard). UNKNOWN - no agent, an older agent's 404, no
crash-guard state - is a normal boot. The decision is logged once ("boot grace ...: ... (R-856)").

NEEDS AN AGENT CHANGE to take effect: GET /host/crash-guard serving the guard's state.json fields
(present, last_boot_at, last_boot_unclean, tripped). Until then every box keeps 90 s.

Tests: TestR856_CrashBootHoldsTheMailsForTheLongGrace, TestR856_NormalBootKeeps90s,
TestR856_FactReadLateInTheNormalGraceStillCounts, TestR856_AgentProbeReadsTheCrashGuardState,
TestR856_CrashGuardDecodesAndAnOlderAgentIs404, TestR856_DeadAppCheckWaitsOnTheCrashAwareGrace,
TestR856_NormalGraceIsTheDeadAppBootGrace.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/cmd/controller/main.go

@@ -35,6 +35,7 @@ import (
 	"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
 	cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/crashboot"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/i18n"
@@ -936,9 +937,23 @@ func main() {
 	// deadAppHeartbeatEvery-th scan emits one INFO carrying the scan count and what it found, so an
 	// operator can always answer "is it running, and what does it see?" from a default box — and a
 	// STALLED detector is visible as the heartbeat stopping.
+	//
+	// R-856 (`09` §3 decision 143, extending 129): after a CRASH boot of the host the grace is about 15
+	// minutes, not 90 s — the 2026-10-04 crash-guard test mailed app_start_failed / app_stopped_unhealthy
+	// 3.5 and 9 minutes after the boot for apps still coming up. The fact is the host crash guard's,
+	// read through the agent (GET /host/crash-guard); unknown (no agent, an older agent's 404) keeps 90 s.
+	var crashProbe crashboot.Probe
+	if cfg.LocalAPI.Endpoint != "" && cfg.LocalAPI.Token != "" {
+		if ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint); err != nil {
+			logger.Printf("[WARN] [deadapp] agent client init failed (%v) — the boot grace cannot learn of a crash boot; 90 s", err)
+		} else {
+			crashProbe = crashboot.AgentProbe(ac.CrashGuard)
+		}
+	}
+	bootGrace := crashboot.New(startTime, crashProbe, logger)
 	sched.Every("deadapp-check", 30*time.Second, func(ctx context.Context) error {
-		if time.Since(startTime) < deadAppBootGrace {
-			return nil // still inside the startup settle window
+		if bootGrace.Within(time.Now()) {
+			return nil // still inside the startup settle window (90 s, or ~15 min after a crash boot)
 		}
 		dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard, backupMgr)
 		alertMgr.SetDeadAppAlerts(dead)

controller/cmd/controller/r856_wiring_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/crashboot"
+)
+
+// R-856 (`09` §3 decision 143): the dead-app check — the source of app_start_failed and
+// app_stopped_unhealthy — waits on the crash-aware boot grace, not on the fixed 90 s constant.
+//
+// Red-proof: put `time.Since(startTime) < deadAppBootGrace` back as the check's gate — this test fails.
+func TestR856_DeadAppCheckWaitsOnTheCrashAwareGrace(t *testing.T) {
+	f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var body *ast.BlockStmt
+	ast.Inspect(f, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok || len(call.Args) < 3 {
+			return true
+		}
+		if sel, ok := call.Fun.(*ast.SelectorExpr); !ok || sel.Sel.Name != "Every" {
+			return true
+		}
+		if lit, ok := call.Args[0].(*ast.BasicLit); ok && lit.Value == `"deadapp-check"` {
+			if fl, ok := call.Args[2].(*ast.FuncLit); ok {
+				body = fl.Body
+			}
+		}
+		return true
+	})
+	if body == nil {
+		t.Fatal("the deadapp-check schedule is gone from main.go")
+	}
+	within, constant := false, false
+	ast.Inspect(body, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.SelectorExpr:
+			if id, ok := x.X.(*ast.Ident); ok && id.Name == "bootGrace" && x.Sel.Name == "Within" {
+				within = true
+			}
+		case *ast.Ident:
+			if x.Name == "deadAppBootGrace" {
+				constant = true
+			}
+		}
+		return true
+	})
+	if !within || constant {
+		t.Errorf("the dead-app check must gate on bootGrace.Within (found=%v) and not on deadAppBootGrace (found=%v)", within, constant)
+	}
+}
+
+// The normal grace is today's 90 s: a normal boot must keep it exactly (decision 143's other half).
+func TestR856_NormalGraceIsTheDeadAppBootGrace(t *testing.T) {
+	if crashboot.NormalGrace != deadAppBootGrace {
+		t.Errorf("crashboot.NormalGrace = %s, deadAppBootGrace = %s — a normal boot must keep today's grace", crashboot.NormalGrace, deadAppBootGrace)
+	}
+}