Files
felhom.eu/hub/internal/notify/dispatcher_test.go
T
admin 7f11cfb36c hub v0.65.0 — PBS DR storage visibility (ep0 usage op) + Offsite tab split + dual dashboard gauges (R-5)
Makes PBS DR storage visible like the restic pool box (v0.64.0), differentiated. Scoping
correction: restic = subaccounts on the shared Hetzner Storage Box (Hetzner API); PBS DR =
the felhom-offsite PBS datastore on the ep0 endpoint VM (NO Hetzner API). Option A
(Viktor-ruled): a read-only `usage` op on the felhom-tenantsync ep0 forced command (twin of
fingerprint), polled by a new hub checker on the 15-min throttle. READ-ONLY throughout.

Phase-0 (gate PASSED): on ep0 (PBS 4.2.3), df -B1 --output=size,used,avail <datastore path>
yields bytes (39990112256/7627939840/... ~19%), read-only, existing sudo context, no admin token.

- scripts/felhom-tenantsync.sh -> v1.2.0: read-only `usage` short-circuit (df on the datastore
  path), no customer_id, no admin token, NO mutation. + a bash harness proving zero mutation.
- tenantsync.Client.Usage() + BoxUsage; unknown-op -> typed ErrUsageUnsupported (graceful).
- monitor.PBSDRBoxChecker: OffsiteBoxChecker clone over a usageReader seam; 15-min throttle,
  cached PBSBoxSnapshot, escalation-only pbsdr_box_fill on the "pbsdr-box" scope (operator only,
  no SaveEvent), recovery re-arm. Fill only. THREE states: ok / unavailable (ep0 <=v1.1.0,
  neutral no-alert) / degraded (exec failed, keep last).
- config: Alerting.PBSDRBoxFill{Warn,Crit}Percent (80/90); built with the tenantsync client,
  60s sweep, SetPBSDRBox. Hub deploy INDEPENDENT of the ep0 update (graceful degradation).
- web: /offsite splits into Restic + PBS DR hash tabs (endpoint cards under PBS DR); PBS panel;
  the single dashboard tile becomes two gauges (RESTIC pct.ratio, PBS DR pct / n/a).
- runbook offsite-endpoint.md 10: v1.2.0 update steps (no sudoers/authorized_keys change).

Tests: 10 Go + the harness; 3 red-proofs (usage mutation, escalation-only, unavailable-drives-band)
confirmed red then restored. go build/vet/test + bash -n + hub confirm gate all pass.
2026-07-17 21:13:30 +02:00

131 lines
5.0 KiB
Go

package notify
import (
"bytes"
"io"
"log"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
_ "modernc.org/sqlite"
)
func newDispStore(t *testing.T) *store.Store {
t.Helper()
st, err := store.New(filepath.Join(t.TempDir(), "d.db"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"})
return st
}
// TestSeverityNotifies is the Part-0 contract + companion red-proof: warning/error/critical notify; info
// and unknown values do not. The companion models the PRE-FIX predicate (warning||error) that silently
// dropped "critical" — the bug this fixes.
func TestSeverityNotifies(t *testing.T) {
for _, s := range []string{"warning", "error", "critical"} {
if !severityNotifies(s) {
t.Errorf("%q must notify", s)
}
}
for _, s := range []string{"info", "", "frobnicate", "debug"} {
if severityNotifies(s) {
t.Errorf("%q must NOT notify", s)
}
}
// COMPANION RED-PROOF: the pre-fix predicate dropped "critical". The fix routes it.
preFix := func(sev string) bool { return sev == "warning" || sev == "error" }
if preFix("critical") {
t.Fatal("control: the pre-fix predicate should not match critical")
}
if !severityNotifies("critical") {
t.Fatal("the fix MUST route critical where the old (warning||error) predicate dropped it")
}
}
// TestProcessEvent_PoolBoxScopeOperatorOnly (v0.64.0, R-5, Scenario C7): a customer-less "pool-box"
// event reaches ONLY the operator channel — processCustomer finds no prefs for it and no-ops, so no
// customer email is ever attempted for the synthetic scope.
func TestProcessEvent_PoolBoxScopeOperatorOnly(t *testing.T) {
st := newDispStore(t)
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
var mu sync.Mutex
var sent []string
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
d.ProcessEvent("pool-box", "offsite_box_fill", "warning", "Offsite pool box 82% full", `{"scope":"pool-box"}`, "hub")
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
t.Fatalf("a pool-box event must reach the operator ONLY (no customer email), sent=%v", sent)
}
}
// TestProcessEvent_PBSDRBoxScopeOperatorOnly (v0.65.0, R-5, Scenario C3): a customer-less "pbsdr-box"
// event reaches ONLY the operator channel (same property as pool-box — no prefs → no customer email).
func TestProcessEvent_PBSDRBoxScopeOperatorOnly(t *testing.T) {
st := newDispStore(t)
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
var mu sync.Mutex
var sent []string
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
d.ProcessEvent("pbsdr-box", "pbsdr_box_fill", "warning", "PBS DR datastore 82% full", `{"scope":"pbsdr-box"}`, "hub")
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
t.Fatalf("a pbsdr-box event must reach the operator ONLY, sent=%v", sent)
}
}
// TestProcessEvent_CriticalRoutes: a critical-severity event reaches the operator channel (Scenario E).
func TestProcessEvent_CriticalRoutes(t *testing.T) {
st := newDispStore(t)
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
var mu sync.Mutex
var sent []string
d.sendEmailFn = func(to, _, _ string) error { mu.Lock(); defer mu.Unlock(); sent = append(sent, to); return nil }
d.ProcessEvent("c1", "host_disk_critical", "critical", "root full", "{}", "hub")
if len(sent) != 1 || sent[0] != "op@felhom.eu" {
t.Fatalf("critical must route to the operator, sent=%v", sent)
}
}
// TestProcessEvent_UnknownSeverityLogged: an unrecognized severity is LOGGED (not silently dropped) and
// not routed.
func TestProcessEvent_UnknownSeverityLogged(t *testing.T) {
st := newDispStore(t)
var buf bytes.Buffer
d := NewDispatcher(st, "test-key", "from", "op@felhom.eu", true, log.New(&buf, "", 0))
sent := 0
d.sendEmailFn = func(_, _, _ string) error { sent++; return nil }
d.ProcessEvent("c1", "weird_event", "frobnicate", "msg", "{}", "hub")
if sent != 0 {
t.Fatal("an unknown severity must not route")
}
if !strings.Contains(buf.String(), "unrecognized severity") {
t.Fatalf("an unknown severity must be logged, got: %q", buf.String())
}
}
// TestProcessEvent_InfoSilent: "info" is an intentional non-notify — not routed AND not logged as
// unrecognized (it's a known, deliberate non-alert).
func TestProcessEvent_InfoSilent(t *testing.T) {
st := newDispStore(t)
var buf bytes.Buffer
d := NewDispatcher(st, "test-key", "from", "op@felhom.eu", true, log.New(&buf, "", 0))
sent := 0
d.sendEmailFn = func(_, _, _ string) error { sent++; return nil }
d.ProcessEvent("c1", "controller_started", "info", "msg", "{}", "hub")
if sent != 0 {
t.Fatal("info must not route")
}
if strings.Contains(buf.String(), "unrecognized") {
t.Fatal("info is an intentional non-notify — it must NOT be logged as unrecognized")
}
}