Files
felhom.eu/hub/internal/monitor/deliberate_off_test.go
T
admin a6acb0f762
gates / gates (push) Successful in 6m27s
Hub v0.149.0: clamp a future deliberate-off time; hold a customer only when all its hosts are off (R-998)
Security review of 50e47e29: requested_at is the box's clock — a future value is cut to the hub's
now; CustomerDeliberatelyOff now needs every host of the customer held (one machine of two no
longer silences the other), and fails closed on a read error or no host. Both red-proofed.
2026-10-13 19:52:12 +02:00

188 lines
5.9 KiB
Go

package monitor
// R-998 (doc 14) — A BOX THE HOUSEHOLD SWITCHED OFF IS NOT A BOX THAT DIED, FOR 14 DAYS.
//
// The consequence is asserted, not the mechanism: does the alarm fire? Each "held" case runs beside a CONTROL
// with the same silence and no power-off, which must alarm — a quiet result alone could mean a broken fixture.
import (
"database/sql"
"io"
"log"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
type offRig struct {
st *store.Store
path string
node *StalenessChecker
host *HostStalenessChecker
events []string
msgs []string
}
// newOffRig: one customer with one host, both reporting NOW, both checkers having seen them healthy.
func newOffRig(t *testing.T) *offRig {
t.Helper()
st, path := seedStalenessCustomer(t, "ok", 0)
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostReport("h1", "c1", []byte(`{}`), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
r := &offRig{st: st, path: path}
on := func(cid, et, sev, msg, det, src string) {
r.events = append(r.events, et)
r.msgs = append(r.msgs, msg)
}
r.node = NewStalenessChecker(st, 30*time.Minute, on, log.New(io.Discard, "", 0))
r.host = NewHostStalenessChecker(st, 30*time.Minute, on, log.New(io.Discard, "", 0))
r.node.Check()
r.host.Check()
if len(r.events) != 0 {
t.Fatalf("setup emitted %v", r.events)
}
return r
}
// silent ages both the controller's and the agent's last report by d.
func (r *offRig) silent(t *testing.T, d time.Duration) {
t.Helper()
db, err := sql.Open("sqlite", r.path)
if err != nil {
t.Fatal(err)
}
defer db.Close()
at := time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05")
if _, err := db.Exec(`UPDATE reports SET received_at = ? WHERE customer_id = 'c1'`, at); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`UPDATE hosts SET last_report_at = ? WHERE host_id = 'h1'`, at); err != nil {
t.Fatal(err)
}
}
func (r *offRig) offAt(t *testing.T, ago time.Duration) {
t.Helper()
at := time.Now().UTC().Add(-ago).Truncate(time.Second)
if _, err := r.st.SetHostDeliberateOff("h1", &at); err != nil {
t.Fatal(err)
}
}
func (r *offRig) check() { r.node.Check(); r.host.Check() }
func hasEvent(evs []string, want string) bool {
for _, e := range evs {
if e == want {
return true
}
}
return false
}
// Silent for 3 hours: the control alarms node_down + host_down; the switched-off box raises nothing.
func TestDeliberateOff_NoSilenceAlarmDuringTheHold(t *testing.T) {
ctl := newOffRig(t)
ctl.silent(t, 3*time.Hour)
ctl.check()
if !hasEvent(ctl.events, "node_down") || !hasEvent(ctl.events, "host_down") {
t.Fatalf("CONTROL: a box silent 3 h must alarm, got %v — the fixture is broken", ctl.events)
}
r := newOffRig(t)
r.offAt(t, 3*time.Hour)
r.silent(t, 3*time.Hour)
r.check()
r.check()
if len(r.events) != 0 {
t.Fatalf("a box the household switched off alarmed: %v", r.events)
}
if r.node.GetState("c1") != StateDeliberateOff || r.host.GetState("h1") != "off" {
t.Fatalf("states node=%q host=%q, want %q / off", r.node.GetState("c1"), r.host.GetState("h1"), StateDeliberateOff)
}
}
// After the 14-day hold, still silent → the usual alarms fire, and their text names the power-off.
func TestDeliberateOff_AlarmsAfterFourteenDays(t *testing.T) {
r := newOffRig(t)
r.offAt(t, 13*24*time.Hour)
r.silent(t, 13*24*time.Hour)
r.check()
if len(r.events) != 0 {
t.Fatalf("day 13: %v", r.events)
}
r.offAt(t, 14*24*time.Hour+time.Minute) // the hold ran out
r.silent(t, 14*24*time.Hour+time.Minute)
r.check()
if !hasEvent(r.events, "node_down") || !hasEvent(r.events, "host_down") {
t.Fatalf("after 14 days: events %v, want node_down + host_down", r.events)
}
for _, m := range r.msgs {
if !strings.Contains(m, "household") {
t.Fatalf("the alarm does not say the household switched it off: %q", m)
}
}
r.check() // once, not every minute
if len(r.events) != 2 {
t.Fatalf("repeated: %v", r.events)
}
}
// Switched back on inside the hold: reports resume → no recovery mail for an outage that never was.
func TestDeliberateOff_BackOnIsQuiet(t *testing.T) {
r := newOffRig(t)
r.offAt(t, 2*time.Hour)
r.silent(t, 2*time.Hour)
r.check()
r.silent(t, 0)
if _, err := r.st.SetHostDeliberateOff("h1", nil); err != nil { // the new boot's first report
t.Fatal(err)
}
r.check()
if len(r.events) != 0 {
t.Fatalf("back on: %v", r.events)
}
if r.node.GetState("c1") != "ok" || r.host.GetState("h1") != "ok" {
t.Fatalf("states node=%q host=%q, want ok", r.node.GetState("c1"), r.host.GetState("h1"))
}
}
// The 05:00 deadline: Tester 2's shape (R-872) alarms; the same box switched off by the household does not.
func TestDeliberateOff_DeadlineSkips(t *testing.T) {
st, _, sc := downBox(t, 5*24*time.Hour)
if got := deadlineEvents(st, sc); got["expected_backup_missed"] != 1 {
t.Fatalf("CONTROL: %v — the down box must raise its missed-backup alarm", got)
}
st2, _, sc2 := downBox(t, 5*24*time.Hour)
at := time.Now().UTC().Add(-2 * time.Hour)
if _, err := st2.SetHostDeliberateOff("h1", &at); err != nil {
t.Fatal(err)
}
if got := deadlineEvents(st2, sc2); len(got) != 0 {
t.Fatalf("a switched-off box got deadline alarms: %v", got)
}
}
// One switched-off machine must not hold the alarms of the customer's OTHER machine (security review 2026-10-13).
func TestDeliberateOff_OneOfTwoHostsDoesNotHoldTheCustomer(t *testing.T) {
r := newOffRig(t)
if err := r.st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c1", APIKey: "k2"}); err != nil {
t.Fatal(err)
}
r.offAt(t, 3*time.Hour) // only h1 is switched off
if _, held := r.st.CustomerDeliberatelyOff("c1", time.Now()); held {
t.Fatal("the customer is held although its second machine was never switched off")
}
r.silent(t, 3*time.Hour)
r.node.Check()
if !hasEvent(r.events, "node_down") {
t.Fatalf("the customer-level alarm was held by one machine of two: %v", r.events)
}
}