R-856: after a crash boot of the host, app mails wait ~15 minutes; a normal boot keeps 90 s (09 decision 143)

The dead-app check (source of app_start_failed and app_stopped_unhealthy) now gates on a crash-aware
boot grace (internal/crashboot): 15 min when the host crash guard's last boot was UNCLEAN and within
30 min of the controller start, otherwise 90 s. The fact is read from the agent's local API
(GET /host/crash-guard, agentapi.Client.CrashGuard). UNKNOWN - no agent, an older agent's 404, no
crash-guard state - is a normal boot. The decision is logged once ("boot grace ...: ... (R-856)").

NEEDS AN AGENT CHANGE to take effect: GET /host/crash-guard serving the guard's state.json fields
(present, last_boot_at, last_boot_unclean, tripped). Until then every box keeps 90 s.

Tests: TestR856_CrashBootHoldsTheMailsForTheLongGrace, TestR856_NormalBootKeeps90s,
TestR856_FactReadLateInTheNormalGraceStillCounts, TestR856_AgentProbeReadsTheCrashGuardState,
TestR856_CrashGuardDecodesAndAnOlderAgentIs404, TestR856_DeadAppCheckWaitsOnTheCrashAwareGrace,
TestR856_NormalGraceIsTheDeadAppBootGrace.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/crashboot/crashboot_test.go

@@ -0,0 +1,106 @@
+package crashboot
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"log"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
+)
+
+// R-856 (`09` §3 decision 143): after a CRASH boot the app mails wait about 15 minutes; a normal boot
+// keeps 90 s; unknown is a normal boot.
+//
+// COMPANION RED-PROOF: in tryResolve's default branch, leave g.crashBoot false — the crash-boot test
+// then reads Within(start+10m) = false (mails would go out at 10 minutes), and fails.
+
+var start = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
+
+func fixed(f Fact, err error) Probe {
+	return func(context.Context) (Fact, error) { return f, err }
+}
+
+// The 2026-10-04 shape: the host crashed, came back, and the controller started ~1 minute later. The
+// mails came at 3.5 and 9 minutes after the boot — both must now be inside the grace.
+func TestR856_CrashBootHoldsTheMailsForTheLongGrace(t *testing.T) {
+	var logs bytes.Buffer
+	g := New(start, fixed(Fact{Known: true, Unclean: true, BootAt: start.Add(-time.Minute)}, nil), log.New(&logs, "", 0))
+	for _, at := range []time.Duration{30 * time.Second, 100 * time.Second, 210 * time.Second, 9 * time.Minute, 14 * time.Minute} {
+		if !g.Within(start.Add(at)) {
+			t.Errorf("crash boot: at +%s the mails must still wait (grace %s)", at, g.Duration(start.Add(at)))
+		}
+	}
+	if g.Within(start.Add(CrashGrace + time.Second)) {
+		t.Error("crash boot: after the long grace the mails must flow again")
+	}
+	if !strings.Contains(logs.String(), "boot grace 15m0s") || !strings.Contains(logs.String(), "UNCLEAN") {
+		t.Errorf("the decision must be logged once, positively; log:\n%s", logs.String())
+	}
+}
+
+func TestR856_NormalBootKeeps90s(t *testing.T) {
+	for name, p := range map[string]Probe{
+		"clean boot":        fixed(Fact{Known: true, Unclean: false, BootAt: start.Add(-time.Minute)}, nil),
+		"old unclean boot":  fixed(Fact{Known: true, Unclean: true, BootAt: start.Add(-48 * time.Hour)}, nil),
+		"unclean, no time":  fixed(Fact{Known: true, Unclean: true}, nil),
+		"agent predates it": fixed(Fact{}, errors.New("agentapi: GET /host/crash-guard: HTTP 404")),
+		"no crash guard":    fixed(Fact{Known: false}, nil),
+		"no agent (nil)":    nil,
+	} {
+		g := New(start, p, nil)
+		if !g.Within(start.Add(30 * time.Second)) {
+			t.Errorf("%s: inside the normal grace the mails wait", name)
+		}
+		if g.Within(start.Add(NormalGrace + time.Second)) {
+			t.Errorf("%s: a normal or unknown boot must keep today's 90 s grace, got %s", name, g.Duration(start.Add(NormalGrace+time.Second)))
+		}
+	}
+}
+
+// The agent comes up a few seconds after the controller: an error early in the normal grace is not
+// final; the fact read later in it still decides.
+func TestR856_FactReadLateInTheNormalGraceStillCounts(t *testing.T) {
+	calls := 0
+	g := New(start, func(context.Context) (Fact, error) {
+		calls++
+		if calls == 1 {
+			return Fact{}, errors.New("connection refused")
+		}
+		return Fact{Known: true, Unclean: true, BootAt: start.Add(-2 * time.Minute)}, nil
+	}, nil)
+	g.Within(start.Add(30 * time.Second))
+	if !g.Within(start.Add(5 * time.Minute)) {
+		t.Fatal("a crash boot learned on the second ask must still hold the mails")
+	}
+	g.Within(start.Add(6 * time.Minute))
+	if calls != 2 {
+		t.Errorf("once resolved the fact is never asked again, asked %d times", calls)
+	}
+}
+
+// The agent's answer, end to end through the adapter: the state.json shape the crash guard writes.
+func TestR856_AgentProbeReadsTheCrashGuardState(t *testing.T) {
+	boot := start.Add(-time.Minute).Format("2006-01-02T15:04:05Z")
+	g := New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
+		return agentapi.CrashGuardState{Present: true, LastBootAt: boot, LastBootUnclean: true}, nil
+	}), nil)
+	if !g.Within(start.Add(10 * time.Minute)) {
+		t.Error("an unclean boot read through the agent must hold the mails")
+	}
+	g = New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
+		return agentapi.CrashGuardState{Present: false, LastBootUnclean: true}, nil
+	}), nil)
+	if g.Within(start.Add(NormalGrace + time.Second)) {
+		t.Error("a host with no crash-guard state is unknown — a normal boot")
+	}
+	g = New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
+		return agentapi.CrashGuardState{}, &agentapi.StatusError{Path: "/host/crash-guard", Code: 404}
+	}), nil)
+	if g.Within(start.Add(NormalGrace + time.Second)) {
+		t.Error("an agent that predates the route (404) is unknown — a normal boot")
+	}
+}