felhom-controller/controller/internal/crashboot/crashboot_test.go

package crashboot

import (
	"bytes"
	"context"
	"errors"
	"log"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
)

// R-856 (`09` §3 decision 143): after a CRASH boot the app mails wait about 15 minutes; a normal boot
// keeps 90 s; unknown is a normal boot.
//
// COMPANION RED-PROOF: in tryResolve's default branch, leave g.crashBoot false — the crash-boot test
// then reads Within(start+10m) = false (mails would go out at 10 minutes), and fails.

var start = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)

func fixed(f Fact, err error) Probe {
	return func(context.Context) (Fact, error) { return f, err }
}

// The 2026-10-04 shape: the host crashed, came back, and the controller started ~1 minute later. The
// mails came at 3.5 and 9 minutes after the boot — both must now be inside the grace.
func TestR856_CrashBootHoldsTheMailsForTheLongGrace(t *testing.T) {
	var logs bytes.Buffer
	g := New(start, fixed(Fact{Known: true, Unclean: true, BootAt: start.Add(-time.Minute)}, nil), log.New(&logs, "", 0))
	for _, at := range []time.Duration{30 * time.Second, 100 * time.Second, 210 * time.Second, 9 * time.Minute, 14 * time.Minute} {
		if !g.Within(start.Add(at)) {
			t.Errorf("crash boot: at +%s the mails must still wait (grace %s)", at, g.Duration(start.Add(at)))
		}
	}
	if g.Within(start.Add(CrashGrace + time.Second)) {
		t.Error("crash boot: after the long grace the mails must flow again")
	}
	if !strings.Contains(logs.String(), "boot grace 15m0s") || !strings.Contains(logs.String(), "UNCLEAN") {
		t.Errorf("the decision must be logged once, positively; log:\n%s", logs.String())
	}
}

func TestR856_NormalBootKeeps90s(t *testing.T) {
	for name, p := range map[string]Probe{
		"clean boot":        fixed(Fact{Known: true, Unclean: false, BootAt: start.Add(-time.Minute)}, nil),
		"old unclean boot":  fixed(Fact{Known: true, Unclean: true, BootAt: start.Add(-48 * time.Hour)}, nil),
		"unclean, no time":  fixed(Fact{Known: true, Unclean: true}, nil),
		"agent predates it": fixed(Fact{}, errors.New("agentapi: GET /host/crash-guard: HTTP 404")),
		"no crash guard":    fixed(Fact{Known: false}, nil),
		"no agent (nil)":    nil,
	} {
		g := New(start, p, nil)
		if !g.Within(start.Add(30 * time.Second)) {
			t.Errorf("%s: inside the normal grace the mails wait", name)
		}
		if g.Within(start.Add(NormalGrace + time.Second)) {
			t.Errorf("%s: a normal or unknown boot must keep today's 90 s grace, got %s", name, g.Duration(start.Add(NormalGrace+time.Second)))
		}
	}
}

// The agent comes up a few seconds after the controller: an error early in the normal grace is not
// final; the fact read later in it still decides.
func TestR856_FactReadLateInTheNormalGraceStillCounts(t *testing.T) {
	calls := 0
	g := New(start, func(context.Context) (Fact, error) {
		calls++
		if calls == 1 {
			return Fact{}, errors.New("connection refused")
		}
		return Fact{Known: true, Unclean: true, BootAt: start.Add(-2 * time.Minute)}, nil
	}, nil)
	g.Within(start.Add(30 * time.Second))
	if !g.Within(start.Add(5 * time.Minute)) {
		t.Fatal("a crash boot learned on the second ask must still hold the mails")
	}
	g.Within(start.Add(6 * time.Minute))
	if calls != 2 {
		t.Errorf("once resolved the fact is never asked again, asked %d times", calls)
	}
}

// The agent's answer, end to end through the adapter: the state.json shape the crash guard writes.
func TestR856_AgentProbeReadsTheCrashGuardState(t *testing.T) {
	boot := start.Add(-time.Minute).Format("2006-01-02T15:04:05Z")
	g := New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
		return agentapi.CrashGuardState{Present: true, LastBootAt: boot, LastBootUnclean: true}, nil
	}), nil)
	if !g.Within(start.Add(10 * time.Minute)) {
		t.Error("an unclean boot read through the agent must hold the mails")
	}
	g = New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
		return agentapi.CrashGuardState{Present: false, LastBootUnclean: true}, nil
	}), nil)
	if g.Within(start.Add(NormalGrace + time.Second)) {
		t.Error("a host with no crash-guard state is unknown — a normal boot")
	}
	g = New(start, AgentProbe(func(context.Context) (agentapi.CrashGuardState, error) {
		return agentapi.CrashGuardState{}, &agentapi.StatusError{Path: "/host/crash-guard", Code: 404}
	}), nil)
	if g.Within(start.Add(NormalGrace + time.Second)) {
		t.Error("an agent that predates the route (404) is unknown — a normal boot")
	}
}

// R-1007: Decided exposes the same verdict, with the unclean boot's time. Unknown stays unresolved inside the
// normal grace and resolves to NOT a crash at its end.
func TestDecided_CrashCarriesTheBootTimeUnknownResolvesNormal(t *testing.T) {
	boot := start.Add(-time.Minute)
	g := New(start, fixed(Fact{Known: true, Unclean: true, BootAt: boot}, nil), nil)
	if res, crash, at := g.Decided(start.Add(time.Second)); !res || !crash || !at.Equal(boot) {
		t.Fatalf("crash: resolved=%t crash=%t at=%s", res, crash, at)
	}
	u := New(start, fixed(Fact{}, nil), nil)
	if res, _, _ := u.Decided(start.Add(10 * time.Second)); res {
		t.Fatal("unknown must not resolve inside the normal grace (the agent may come up late)")
	}
	if res, crash, _ := u.Decided(start.Add(NormalGrace)); !res || crash {
		t.Fatalf("unknown at the end of the normal grace: resolved=%t crash=%t, want a normal boot", res, crash)
	}
}