felhom.eu/hub/internal/monitor/staleness_disabled_test.go

package monitor

// R-321 — A MACHINE WE TOLD TO BE QUIET IS NOT A MACHINE THAT DIED.
//
// Switching a box's hub reporting off is a supported product state. The controller sends one final
// report carrying health.status = "disabled" and then goes quiet BY DESIGN. The hub parses and stores
// that status, and the operator roll-up already renders such a customer as `disabled` — but this
// checker measured only the AGE of the last report, so the machine went stale at 30 minutes, down at
// 60, and e-mailed the operator twice about an outage we caused on purpose.
//
// The discriminator is the box's OWN last word, not an inference, and it is in the data this checker
// already reads (`CustomerSummary.HealthStatus`). Suppressing on a guess would be worse than the
// false alarm it removes; suppressing on the machine's own declaration is not a guess.

import (
	"database/sql"
	"io"
	"log"
	"path/filepath"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	_ "modernc.org/sqlite"
)

// seedStalenessCustomer creates a store with one customer and one report of the given health status,
// backdated by `age`. Reports are inserted through the REAL SaveReport path so the health_status
// denormalization is exercised rather than hand-set — a test that writes the column directly cannot
// see a decode that never happens, which is the R-260 class.
func seedStalenessCustomer(t *testing.T, health string, age time.Duration) (*store.Store, string) {
	t.Helper()
	path := filepath.Join(t.TempDir(), "t.db")
	st, err := store.New(path, log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatalf("store.New: %v", err)
	}
	t.Cleanup(func() { st.Close() })
	if err := st.SaveCustomerConfig(&store.CustomerConfig{
		CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p", Status: "active",
	}); err != nil {
		t.Fatalf("SaveCustomerConfig: %v", err)
	}
	saveReportAged(t, st, path, health, age)
	return st, path
}

// saveReportAged saves a controller report with the given health status and then backdates its
// received_at. The backdate is a raw UPDATE because there is no production path that writes an old
// timestamp — which is exactly why it is confined to this one helper.
func saveReportAged(t *testing.T, st *store.Store, path, health string, age time.Duration) {
	t.Helper()
	body := `{"customer_id":"c1","health":{"status":"` + health + `"}}`
	if err := st.SaveReport("c1", []byte(body)); err != nil {
		t.Fatalf("SaveReport: %v", err)
	}
	if age <= 0 {
		return
	}
	// Backdating goes through a SECOND connection to the same file rather than a test-only method on
	// Store. No production path writes an old received_at, so exposing one would widen the store's
	// API for a fixture — and this keeps the ageing confined to the one helper that needs it.
	db, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatalf("open for backdate: %v", err)
	}
	defer db.Close()
	// GetCustomers selects the report with MAX(received_at), not MAX(id) — so backdating only the
	// row just written would leave an EARLIER, fresher-looking row as the one the checker reads, and
	// the fixture would quietly test the wrong report. (Found by two scenarios failing on a change
	// that was correct: the instrument was wrong, not the code.) Every older row is therefore pushed
	// further back, so the row just written is unambiguously the latest.
	when := time.Now().UTC().Add(-age)
	newest := when.Format("2006-01-02 15:04:05")
	older := when.Add(-time.Hour).Format("2006-01-02 15:04:05")
	if _, err := db.Exec(
		`UPDATE reports SET received_at = ? WHERE customer_id = 'c1' AND id < (SELECT MAX(id) FROM reports WHERE customer_id = 'c1')`,
		older); err != nil {
		t.Fatalf("age the older rows: %v", err)
	}
	if _, err := db.Exec(
		`UPDATE reports SET received_at = ? WHERE id = (SELECT MAX(id) FROM reports WHERE customer_id = 'c1')`,
		newest); err != nil {
		t.Fatalf("backdate: %v", err)
	}
}

// newChecker builds a checker with a 30-minute threshold and records every event it emits.
func newChecker(t *testing.T, st *store.Store) (*StalenessChecker, *[]string) {
	t.Helper()
	var events []string
	sc := NewStalenessChecker(st, 30*time.Minute, func(cid, et, sev, msg, det, src string) {
		events = append(events, et)
	}, log.New(io.Discard, "", 0))
	return sc, &events
}

// ── A — a machine deliberately silent for days ──────────────────────────────────────────────────
//
// WRONG OUTCOME GUARDED: stale, then down, then two e-mails — which is what happens today.
func TestStaleness_A_DeliberatelySilentDoesNotAlarm(t *testing.T) {
	// The box is HEALTHY and OBSERVED first, then switched off. Seeding it already-disabled would
	// make this test pass vacuously: the checker's new-customer branch sets the first state without
	// an event, so a customer that was never seen healthy can never emit a transition — and the
	// assertion below would hold even with the suppression deleted. (Confirmed: it did. The
	// red-proof for this scenario passed on the state assertion alone until this line was added.)
	st, path := seedStalenessCustomer(t, "ok", 0)
	sc, events := newChecker(t, st)
	sc.Check()
	if sc.GetState("c1") != "ok" {
		t.Fatalf("setup: the machine should start observed-healthy, got %q", sc.GetState("c1"))
	}

	// Reporting is switched off. The box says so on the way out, then goes quiet for three days.
	saveReportAged(t, st, path, "disabled", 72*time.Hour)

	for i := 0; i < 3; i++ { // several passes: a suppression that only holds once is not a suppression
		sc.Check()
	}

	if len(*events) != 0 {
		t.Fatalf("a deliberately-disabled machine emitted %v — three days quiet BY REQUEST", *events)
	}
	// The silence must be VISIBLE, not merely un-alarmed: quiet-on-purpose and quiet-by-accident
	// must not look identical, and an unknown is never drawn as healthy.
	if got := sc.GetState("c1"); got != StateDisabled {
		t.Errorf("state = %q, want %q — the deliberate silence is invisible", got, StateDisabled)
	}
}

// ── B — a machine that simply stopped reporting ─────────────────────────────────────────────────
//
// WRONG OUTCOME GUARDED: silence mistaken for a deliberate switch-off — THE ALARM THAT MATTERS,
// suppressed. This is the one that must never regress.
func TestStaleness_B_GenuineSilenceStillAlarms(t *testing.T) {
	st, path := seedStalenessCustomer(t, "ok", 0)
	sc, events := newChecker(t, st)
	sc.Check() // first observation: healthy, no event

	// The same customer now goes quiet for real.
	saveReportAged(t, st, path, "ok", 3*time.Hour)
	sc.Check()

	if got := sc.GetState("c1"); got != "down" {
		t.Fatalf("a genuinely silent machine is %q, want \"down\" — a real alarm was swallowed", got)
	}
	if len(*events) == 0 {
		t.Fatal("a genuinely silent machine emitted NO event — the suppression is over-broad")
	}
}

// ── C — a disabled machine that is re-enabled and reports promptly ──────────────────────────────
//
// WRONG OUTCOME GUARDED: a recovery e-mail for an outage that never happened.
func TestStaleness_C_ReEnabledReportsCleanlyWithNoRecoveryEvent(t *testing.T) {
	st, path := seedStalenessCustomer(t, "disabled", 72*time.Hour)
	sc, events := newChecker(t, st)
	sc.Check()
	if sc.GetState("c1") != StateDisabled {
		t.Fatalf("setup: expected the disabled state, got %q", sc.GetState("c1"))
	}

	// Reporting is switched back on and the box reports immediately.
	saveReportAged(t, st, path, "ok", 0)
	sc.Check()

	if len(*events) != 0 {
		t.Fatalf("re-enabling emitted %v — there was no outage to recover from", *events)
	}
	if got := sc.GetState("c1"); got != "ok" {
		t.Errorf("state after re-enable = %q, want \"ok\"", got)
	}
}

// ── D — a disabled machine that is re-enabled and then goes genuinely quiet ─────────────────────
//
// WRONG OUTCOME GUARDED: permanently silenced because it was once disabled. Also pins THE CLOCK
// JUDGEMENT: timing runs from the report the box sent on re-enabling, not from the last report
// before the switch-off — otherwise coming back would fire an instant false alarm about a quiet
// period we asked for.
func TestStaleness_D_ReEnabledThenQuietAlarmsFromReEnablement(t *testing.T) {
	st, path := seedStalenessCustomer(t, "disabled", 72*time.Hour)
	sc, events := newChecker(t, st)
	sc.Check()

	// Re-enabled, reports promptly — the clean first observation.
	saveReportAged(t, st, path, "ok", 0)
	sc.Check()
	if len(*events) != 0 {
		t.Fatalf("the re-enable itself emitted %v", *events)
	}

	// …and then goes quiet for real. Timed from THIS report, it is down.
	saveReportAged(t, st, path, "ok", 3*time.Hour)
	sc.Check()

	if got := sc.GetState("c1"); got != "down" {
		t.Fatalf("state = %q, want \"down\" — a once-disabled machine was silenced for ever", got)
	}
	if len(*events) == 0 {
		t.Fatal("no event for a machine that genuinely died after being re-enabled")
	}
}

// The deadline check is a SECOND DOOR onto the same false alarm: a disabled box is not "down", so
// without its own skip it would keep e-mailing expected_backup_missed every morning. R-195's shape,
// which is why both doors are closed together.
func TestStaleness_DisabledIsAlsoSkippedByTheDeadlineCheck(t *testing.T) {
	st, _ := seedStalenessCustomer(t, "disabled", 72*time.Hour)
	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
		t.Fatalf("UpsertHost: %v", err)
	}
	sc, _ := newChecker(t, st)
	sc.Check()

	var events []string
	CheckBackupDeadlines(st, sc, func(cid, et, sev, msg, det, src string) {
		events = append(events, et)
	}, log.New(io.Discard, "", 0))

	for _, e := range events {
		if e == "expected_backup_missed" || e == "expected_dbdump_missed" {
			t.Fatalf("the deadline check alarmed on a deliberately-disabled machine: %v", events)
		}
	}
}