felhom.eu/hub/internal/notify/r389_cooldown_grain_test.go

package notify

import (
	"strings"
	"testing"
)

// R-389 — the operator cooldown named the event TYPE and not the APP, so only the first broken app
// per hour was ever mailed.
//
// THE DEFECT. `processOperator` keys the 1-hour cooldown on
// `customerID:eventType[:tier][:run_id]`. None of those name an app. Measured live on `demo-hp`
// 2026-08-23: `bookstack` alarmed at 09:27:51 and was `sent`; `privatebin` alarmed four minutes
// later and was logged `suppressed — operator cooldown 1h, key=demo-hp:app_start_failed`. Three apps
// dying together produce one mail.
//
// THE LAYER. These sit at the key builder and at `processOperator`. The key is where the collapse
// happens, and the stored notification rows are where it is visible — asserting only that the suffix
// function returns a string would repeat the "mechanism pinned, consequence unpinned" mistake.
//
// RED-PROOF (observed, see REPORT.md): drop `cooldownStackSuffix` from the key expression in
// `processOperator` and TestR389_TwoAppsInsideTheHourBothReachTheOperator fails with
// `2 apps down inside the hour produced 1 operator mail(s), want 2`.

// --- the suffix itself ---------------------------------------------------------------------------

func TestR389_StackSuffixIsAllowListedAndFailSoft(t *testing.T) {
	const appDetails = `{"stack_name":"bookstack","display_name":"BookStack"}`

	cases := []struct {
		name      string
		eventType string
		details   string
		want      string
	}{
		{"the allow-listed type gets the app", "app_start_failed", appDetails, ":bookstack"},

		// THE FENCE. crossdrive_failed is severity `error`, reaches the operator leg, and carries
		// stack_name through CrossDriveDetails — a payload-shape rule would have split it per app and
		// silently undone R-97a/R-182.
		{"crossdrive_failed is NOT split per app", "crossdrive_failed",
			`{"stack_name":"bookstack","method":"rsync"}`, ""},
		{"app_deployed is not in the register", "app_deployed", appDetails, ""},
		{"app_removed is not in the register", "app_removed", appDetails, ""},
		{"backup_failed is not in the register", "backup_failed", appDetails, ""},

		// Fail-soft: a degraded payload must fall back to today's key, never panic, never drop.
		{"empty details", "app_start_failed", "", ""},
		{"no stack_name key", "app_start_failed", `{"display_name":"BookStack"}`, ""},
		{"empty stack_name", "app_start_failed", `{"stack_name":""}`, ""},
		{"malformed JSON that still contains the token", "app_start_failed", `{"stack_name":`, ""},
		{"null details", "app_start_failed", "null", ""},
		{"array instead of object", "app_start_failed", `["stack_name"]`, ""},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := cooldownStackSuffix(tc.eventType, tc.details); got != tc.want {
				t.Fatalf("cooldownStackSuffix(%q, %q) = %q, want %q", tc.eventType, tc.details, got, tc.want)
			}
		})
	}
}

// The register must stay narrow. A new entry is a deliberate act and should fail this until
// someone changes it on purpose, having read the fence.
//
// v0.120.0 widened it ON PURPOSE, by the brief "the undo reaches the fleet" (`09` §3 decision 15):
// an update outcome is one app's event, with no digest behind it — two apps undone on one night are
// two alarms. The backup family stays coarse, as the fence says.
func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
	// v0.121.0 widened it again, on purpose (R-636): an OOM storm is one app's event with no digest.
	// v0.122.0 once more (R-659): a stranded held app is one app's event with no digest.
	// v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest.
	want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"}
	ok := len(perAppCooldownEvents) == len(want)
	for _, w := range want {
		ok = ok && perAppCooldownEvents[w]
	}
	if !ok {
		var got []string
		for k := range perAppCooldownEvents {
			got = append(got, k)
		}
		t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+
			"fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+
			"disk sends one digest, not one mail per app. Read the fence before widening this.", got)
	}
}

// --- Scenario C: the absence claim, WITH its positive control ------------------------------------
//
// "No other event type's key changed" is an absence claim. The control below proves this test can
// SEE a key change first — otherwise a broken key builder would make every row look unchanged and
// the test would pass forever.
func TestR389_NoOtherEventTypeKeyChanges(t *testing.T) {
	// The v0.107.0 key expression, modelled inline. This is the BEFORE value, and modelling it here
	// rather than reading it from git is deliberate: the comparison must survive the file moving.
	oldKey := func(customerID, eventType, details string) string {
		return customerID + ":" + eventType + cooldownTierSuffix(details) + cooldownRunSuffix(details)
	}
	newKey := func(customerID, eventType, details string) string {
		return customerID + ":" + eventType + cooldownTierSuffix(details) + cooldownRunSuffix(details) +
			cooldownStackSuffix(eventType, details)
	}

	// POSITIVE CONTROL FIRST: the pair must be able to differ at all.
	if oldKey("c1", "app_start_failed", `{"stack_name":"bookstack"}`) ==
		newKey("c1", "app_start_failed", `{"stack_name":"bookstack"}`) {
		t.Fatal("the control failed: old and new key agree even for the allow-listed type, so this " +
			"test cannot see a key change and its 'unchanged' verdicts below would be worthless")
	}

	// Every other type — including the ones that carry stack_name — must be byte-identical.
	for _, tc := range []struct{ eventType, details string }{
		{"crossdrive_failed", `{"stack_name":"bookstack","method":"rsync"}`},
		{"crossdrive_completed", `{"stack_name":"docmost"}`},
		{"app_deployed", `{"stack_name":"bookstack","display_name":"BookStack"}`},
		{"app_removed", `{"stack_name":"bookstack"}`},
		{"backup_failed", `{"error":"boom"}`},
		{"backup_run_failures", `{"run_id":"r-42"}`},
		{"whole_guest_backup_failed", `{"tier":"felhom-pbs"}`},
		{"db_dump_failed", `{"stack_name":"docmost"}`},
		{"disk_warning", `{"path":"/mnt/data"}`},
		{"expected_backup_missed", `{"tier":"offsite","run_id":"r-9"}`},
		{"storage_disconnected", `{}`},
		{"health_critical", ""},
	} {
		o := oldKey("demo-hp", tc.eventType, tc.details)
		n := newKey("demo-hp", tc.eventType, tc.details)
		if o != n {
			t.Errorf("%s: cooldown key CHANGED\n  v0.107.0: %s\n  v0.108.0: %s\n"+
				"Only app_start_failed may move. Splitting a backup-family key per app undoes R-97a "+
				"and R-182 — twenty mails where one digest belongs.", tc.eventType, o, n)
		}
	}
}

// --- the CONSEQUENCE, asserted from the stored notification rows --------------------------------

// appEvent builds the details payload the controller actually sends for app_start_failed.
func appEvent(stack string) string {
	return `{"stack_name":"` + stack + `","display_name":"` + strings.ToUpper(stack[:1]) + stack[1:] + `"}`
}

// operatorMails returns the operator-channel rows for a customer, newest first.
func operatorMails(t *testing.T, d *Dispatcher, customerID string) (sent, suppressed int, rows []string) {
	t.Helper()
	entries, err := d.store.GetRecentNotifications(customerID, 50)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		if e.Channel != "operator" || e.EventType != "app_start_failed" {
			continue
		}
		rows = append(rows, e.Status+" | "+e.Message+" | "+e.ErrorMessage)
		switch e.Status {
		case "sent":
			sent++
		case "suppressed":
			suppressed++
		}
	}
	return sent, suppressed, rows
}

// SCENARIO A — two apps go down inside the hour. Both must reach the operator.
//
// This is R-389's whole point, and it asserts the STORED rows, not a return value.
func TestR389_TwoAppsInsideTheHourBothReachTheOperator(t *testing.T) {
	st := opOnlyStore(t)
	rec := &sentTo{}
	d := opOnlyDispatcher(t, st, rec)

	// POSITIVE CONTROL: the operator leg must be able to deliver at all in this configuration,
	// before any count below means anything. (Yesterday's live run could not prove the customer leg
	// because no address was set — do not repeat that shape.)
	d.ProcessEvent("c1", "app_start_failed", "warning",
		"Telepített alkalmazás nem fut: BookStack", appEvent("bookstack"), "controller")
	if sent, _, rows := operatorMails(t, d, "c1"); sent != 1 {
		t.Fatalf("control failed: the FIRST app produced %d sent operator row(s), want 1 — the "+
			"operator leg is not delivering here, so the counts below would be meaningless. rows=%v",
			sent, rows)
	}

	// A different app, same hour, same event type.
	d.ProcessEvent("c1", "app_start_failed", "warning",
		"Telepített alkalmazás nem fut: PrivateBin", appEvent("privatebin"), "controller")

	sent, suppressed, rows := operatorMails(t, d, "c1")
	if sent != 2 {
		t.Fatalf("2 apps down inside the hour produced %d operator mail(s), want 2 "+
			"(suppressed=%d). This is R-389: the second app's alarm took the first app's cooldown "+
			"slot.\nrows: %v", sent, suppressed, rows)
	}
	if suppressed != 0 {
		t.Errorf("a DIFFERENT app was suppressed: %v", rows)
	}

	// And the addresses actually attempted — two distinct deliveries, not one row written twice.
	opCount := 0
	for _, to := range rec.to {
		if to == "operator@felhom.eu" {
			opCount++
		}
	}
	if opCount != 2 {
		t.Errorf("the dispatcher attempted %d operator send(s), want 2 — a stored row without a send "+
			"attempt would be a record of something that did not happen", opCount)
	}
}

// SCENARIO B — the SAME app twice inside the hour. The hour is unchanged: one mail, one suppression.
func TestR389_SameAppTwiceInsideTheHourIsStillSuppressed(t *testing.T) {
	st := opOnlyStore(t)
	rec := &sentTo{}
	d := opOnlyDispatcher(t, st, rec)

	for i := 0; i < 2; i++ {
		d.ProcessEvent("c1", "app_start_failed", "warning",
			"Telepített alkalmazás nem fut: BookStack", appEvent("bookstack"), "controller")
	}

	sent, suppressed, rows := operatorMails(t, d, "c1")
	if sent != 1 || suppressed != 1 {
		t.Fatalf("the same app twice gave sent=%d suppressed=%d, want 1 and 1 — R-389 changes the "+
			"GRAIN, not the hour, and re-alarming the same app is the flood the cooldown exists to "+
			"stop.\nrows: %v", sent, suppressed, rows)
	}
	// The suppression must still name the key, which is what made R-389 findable at all.
	if !strings.Contains(rows[0]+rows[1], "bookstack") {
		t.Errorf("the suppression row does not name the app in its key — that visibility is R-182's "+
			"contribution and is how this defect was found: %v", rows)
	}
}

// SCENARIO D — details missing or malformed. The key degrades to today's and the mail STILL GOES.
func TestR389_DegradedDetailsStillDeliver(t *testing.T) {
	for _, details := range []string{"", "null", `{}`, `{"stack_name":""}`, `{"stack_name":`} {
		st := opOnlyStore(t)
		rec := &sentTo{}
		d := opOnlyDispatcher(t, st, rec)

		d.ProcessEvent("c1", "app_start_failed", "warning", "Telepített alkalmazás nem fut", details, "controller")

		sent, _, rows := operatorMails(t, d, "c1")
		if sent != 1 {
			t.Errorf("details %q: %d operator mail(s), want 1 — a degraded payload must fall back to "+
				"the old key and still deliver. Losing an alarm is worse than mis-routing one. rows=%v",
				details, sent, rows)
		}
	}
}

// A backup-family event carrying stack_name must still collapse — the coarse grain is the design.
func TestR389_CrossdriveStillCollapsesPerHour(t *testing.T) {
	st := opOnlyStore(t)
	rec := &sentTo{}
	d := opOnlyDispatcher(t, st, rec)

	for _, stack := range []string{"bookstack", "docmost", "privatebin"} {
		d.ProcessEvent("c1", "crossdrive_failed", "error",
			"Másodlagos mentés sikertelen: "+stack,
			`{"stack_name":"`+stack+`","method":"rsync"}`, "controller")
	}

	entries, err := d.store.GetRecentNotifications("c1", 50)
	if err != nil {
		t.Fatal(err)
	}
	sent := 0
	for _, e := range entries {
		if e.Channel == "operator" && e.EventType == "crossdrive_failed" && e.Status == "sent" {
			sent++
		}
	}
	if sent != 1 {
		t.Fatalf("three crossdrive_failed events produced %d operator mail(s), want 1 — this family's "+
			"cooldown is coarse ON PURPOSE (R-97a, R-182), and it carries stack_name, so a global "+
			"suffix would have split it into three", sent)
	}
}