felhom.eu/hub/internal/monitor/host_oob_test.go

package monitor

import (
	"io"
	"log"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	_ "modernc.org/sqlite"
)

// oobReport builds a host report's `oob` stanza AS A RELEASED AGENT ACTUALLY SENDS IT.
//
// ⚠ IT DID NOT, UNTIL 2026-08-08 (R-260), and that mattered. This helper omitted
// `operator_key_configured` entirely, so every scenario below was driven by a report shape NO
// RELEASED AGENT PRODUCES — the field and the stanza that carries it shipped together in agent
// v0.72.0. A fixture that is not the wire cannot detect a field missing from the wire's receiver,
// which is one reason the gap survived five weeks under a green suite. Same family as R-262, where
// the cross-repo golden omits the two fields whose drift its key-set test is supposed to guard.
//
// The key defaults to INSTALLED here so the existing scenarios keep their original meaning
// (they vary the service, not the credential); the key's own scenarios live in
// host_oob_operatorkey_test.go, and the never-reported case is exercised by oobReportNoKeyField.
func oobReport(active, reachable, configInvalid, operatorConfigured bool) []byte {
	return oobReportWithKey(active, reachable, configInvalid, operatorConfigured, true)
}

func oobReportWithKey(active, reachable, configInvalid, operatorConfigured, keyInstalled bool) []byte {
	b := func(v bool) string {
		if v {
			return "true"
		}
		return "false"
	}
	return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":` + b(active) +
		`,"felhom_sshd_port":8822,"reachable":` + b(reachable) +
		`,"config_invalid":` + b(configInvalid) +
		`,"operator_peer_configured":` + b(operatorConfigured) +
		`,"operator_key_configured":` + b(keyInstalled) + `}}`)
}

// oobReportNoKeyField is the pre-v0.72.0 shape: an `oob` stanza with no operator_key_configured key
// at all. No released agent sends it; it exists so Scenario F is tested against real JSON rather
// than against a hand-set struct field.
func oobReportNoKeyField() []byte {
	return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":true,"felhom_sshd_port":8822,` +
		`"reachable":true,"config_invalid":false,"operator_peer_configured":true}}`)
}

func newOOBStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { st.Close() })
	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
	return st
}

func TestHostOOB_DegradedThenRecoveredTransitions(t *testing.T) {
	st := newOOBStore(t)
	// healthy at construction (active+reachable, operator configured)
	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
	var events []string
	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
	if c.IsDegraded("h1") {
		t.Fatal("healthy host seeded degraded")
	}

	// felhom-sshd goes DOWN with operator configured → one oob_degraded
	st.SaveHostReport("h1", "c1", oobReport(false, false, false, true), store.HostReportDenorm{})
	c.Check()
	if len(events) != 1 || events[0] != "oob_degraded" {
		t.Fatalf("down+operator-configured → one oob_degraded, got %v", events)
	}
	c.Check() // persistent → no re-alert
	if len(events) != 1 {
		t.Fatalf("persistent degraded must not re-emit, got %v", events)
	}

	// recovers → oob_recovered
	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
	c.Check()
	if len(events) != 2 || events[1] != "oob_recovered" {
		t.Fatalf("recovery → oob_recovered, got %v", events)
	}
}

func TestHostOOB_ConfigInvalidAlerts(t *testing.T) {
	st := newOOBStore(t)
	st.SaveHostReport("h1", "c1", oobReport(true, true, false, false), store.HostReportDenorm{})
	var events []string
	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
	// config invalid (even without operator configured) → degraded
	st.SaveHostReport("h1", "c1", oobReport(true, true, true, false), store.HostReportDenorm{})
	c.Check()
	if len(events) != 1 || events[0] != "oob_degraded" {
		t.Fatalf("config_invalid → oob_degraded, got %v", events)
	}
}

// A box where OOB was NEVER set up (no operator peer) with felhom-sshd down must NOT alert — it's not
// broken, it's unconfigured.
func TestHostOOB_DownButNoOperatorNotDegraded(t *testing.T) {
	st := newOOBStore(t)
	st.SaveHostReport("h1", "c1", oobReport(false, false, false, false), store.HostReportDenorm{})
	var events []string
	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
	c.Check()
	if len(events) != 0 {
		t.Fatalf("unconfigured OOB (no operator peer) must not alert, got %v", events)
	}
}

// A report with no oob stanza (pre-H1 agent) is never evaluated.
func TestHostOOB_NoStanzaIgnored(t *testing.T) {
	st := newOOBStore(t)
	st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
	var events []string
	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
	c.Check()
	if len(events) != 0 {
		t.Fatalf("no oob stanza must not alert, got %v", events)
	}
}

// TestHostOOB_MissingOperatorKey_EndToEnd drives the WHOLE path — raw report JSON → SaveHostReport
// → GetHostOOBStates → the checker → the emitted event — for the box this session exists for.
//
// The per-predicate tests in host_oob_operatorkey_test.go set the row's fields directly, which is
// fine for the verdict but cannot prove the fact SURVIVES THE DECODE. This one can, and it is the
// test that would have failed before R-260.
func TestHostOOB_MissingOperatorKey_EndToEnd(t *testing.T) {
	st := newOOBStore(t)
	// healthy, key installed → seeded clean
	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
	var events []string
	var msgs []string
	c := NewHostOOBChecker(st, func(_, et, _, msg, _, _ string) {
		events = append(events, et)
		msgs = append(msgs, msg)
	}, log.New(io.Discard, "", 0))
	if c.IsDegraded("h1") {
		t.Fatal("healthy host with the key installed seeded degraded")
	}

	// everything still up — only the operator's key is gone
	st.SaveHostReport("h1", "c1", oobReportWithKey(true, true, false, true, false), store.HostReportDenorm{})
	c.Check()
	if len(events) != 1 || events[0] != "oob_degraded" {
		t.Fatalf("a box whose operator key vanished must alert exactly once; got %v", events)
	}
	if !strings.Contains(msgs[0], "authorized_key") {
		t.Errorf("the alert must NAME the missing key — the operator reads this at 07:00 and needs to know which of five things is wrong; got %q", msgs[0])
	}

	// the key comes back → recovered
	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
	c.Check()
	if len(events) != 2 || events[1] != "oob_recovered" {
		t.Fatalf("reinstalling the key must recover; got %v", events)
	}
}

// Scenario F end to end: an `oob` stanza with no operator_key_configured key at all must NOT be a
// silent ok, and must be reported distinctly from a known-missing key.
func TestHostOOB_NoKeyField_IsNotSilentlyOK_EndToEnd(t *testing.T) {
	st := newOOBStore(t)
	st.SaveHostReport("h1", "c1", oobReportNoKeyField(), store.HostReportDenorm{})
	rows, err := st.GetHostOOBStates()
	if err != nil || len(rows) != 1 {
		t.Fatalf("GetHostOOBStates: %v rows=%d", err, len(rows))
	}
	if rows[0].OperatorKeyReported {
		t.Fatal("a stanza with no operator_key_configured decoded as though the agent had reported one")
	}
	if !oobDegraded(rows[0]) {
		t.Fatal("an agent too old to report the key was treated as ok — absence read as a fact, which is this defect through the version door")
	}
	if got := oobDegradedReason(rows[0]); !strings.Contains(got, "too old") {
		t.Errorf("the unknown case must say so distinctly; got %q", got)
	}
}