50e47e29af
gates / gates (push) Successful in 6m40s
hosts.deliberate_off_at follows the agent's deliberate_off field (set, then cleared by the first report of a new boot; one info event each way). While held, every silence alarm is held: host and node staleness, the 05:00 backup deadlines, off-site and restore-test freshness, the OS alarms and the kernel notice. After 14 days the usual alarms fire once, naming the power-off. Pages show KIKAPCSOLVA and the time. Consequence tests with controls; each guard red-proofed.
171 lines
5.2 KiB
Go
171 lines
5.2 KiB
Go
package monitor
|
|
|
|
// R-998 (doc 14) — A BOX THE HOUSEHOLD SWITCHED OFF IS NOT A BOX THAT DIED, FOR 14 DAYS.
|
|
//
|
|
// The consequence is asserted, not the mechanism: does the alarm fire? Each "held" case runs beside a CONTROL
|
|
// with the same silence and no power-off, which must alarm — a quiet result alone could mean a broken fixture.
|
|
|
|
import (
|
|
"database/sql"
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
type offRig struct {
|
|
st *store.Store
|
|
path string
|
|
node *StalenessChecker
|
|
host *HostStalenessChecker
|
|
events []string
|
|
msgs []string
|
|
}
|
|
|
|
// newOffRig: one customer with one host, both reporting NOW, both checkers having seen them healthy.
|
|
func newOffRig(t *testing.T) *offRig {
|
|
t.Helper()
|
|
st, path := seedStalenessCustomer(t, "ok", 0)
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.SaveHostReport("h1", "c1", []byte(`{}`), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := &offRig{st: st, path: path}
|
|
on := func(cid, et, sev, msg, det, src string) {
|
|
r.events = append(r.events, et)
|
|
r.msgs = append(r.msgs, msg)
|
|
}
|
|
r.node = NewStalenessChecker(st, 30*time.Minute, on, log.New(io.Discard, "", 0))
|
|
r.host = NewHostStalenessChecker(st, 30*time.Minute, on, log.New(io.Discard, "", 0))
|
|
r.node.Check()
|
|
r.host.Check()
|
|
if len(r.events) != 0 {
|
|
t.Fatalf("setup emitted %v", r.events)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// silent ages both the controller's and the agent's last report by d.
|
|
func (r *offRig) silent(t *testing.T, d time.Duration) {
|
|
t.Helper()
|
|
db, err := sql.Open("sqlite", r.path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
at := time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05")
|
|
if _, err := db.Exec(`UPDATE reports SET received_at = ? WHERE customer_id = 'c1'`, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.Exec(`UPDATE hosts SET last_report_at = ? WHERE host_id = 'h1'`, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (r *offRig) offAt(t *testing.T, ago time.Duration) {
|
|
t.Helper()
|
|
at := time.Now().UTC().Add(-ago).Truncate(time.Second)
|
|
if _, err := r.st.SetHostDeliberateOff("h1", &at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (r *offRig) check() { r.node.Check(); r.host.Check() }
|
|
|
|
func hasEvent(evs []string, want string) bool {
|
|
for _, e := range evs {
|
|
if e == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Silent for 3 hours: the control alarms node_down + host_down; the switched-off box raises nothing.
|
|
func TestDeliberateOff_NoSilenceAlarmDuringTheHold(t *testing.T) {
|
|
ctl := newOffRig(t)
|
|
ctl.silent(t, 3*time.Hour)
|
|
ctl.check()
|
|
if !hasEvent(ctl.events, "node_down") || !hasEvent(ctl.events, "host_down") {
|
|
t.Fatalf("CONTROL: a box silent 3 h must alarm, got %v — the fixture is broken", ctl.events)
|
|
}
|
|
|
|
r := newOffRig(t)
|
|
r.offAt(t, 3*time.Hour)
|
|
r.silent(t, 3*time.Hour)
|
|
r.check()
|
|
r.check()
|
|
if len(r.events) != 0 {
|
|
t.Fatalf("a box the household switched off alarmed: %v", r.events)
|
|
}
|
|
if r.node.GetState("c1") != StateDeliberateOff || r.host.GetState("h1") != "off" {
|
|
t.Fatalf("states node=%q host=%q, want %q / off", r.node.GetState("c1"), r.host.GetState("h1"), StateDeliberateOff)
|
|
}
|
|
}
|
|
|
|
// After the 14-day hold, still silent → the usual alarms fire, and their text names the power-off.
|
|
func TestDeliberateOff_AlarmsAfterFourteenDays(t *testing.T) {
|
|
r := newOffRig(t)
|
|
r.offAt(t, 13*24*time.Hour)
|
|
r.silent(t, 13*24*time.Hour)
|
|
r.check()
|
|
if len(r.events) != 0 {
|
|
t.Fatalf("day 13: %v", r.events)
|
|
}
|
|
r.offAt(t, 14*24*time.Hour+time.Minute) // the hold ran out
|
|
r.silent(t, 14*24*time.Hour+time.Minute)
|
|
r.check()
|
|
if !hasEvent(r.events, "node_down") || !hasEvent(r.events, "host_down") {
|
|
t.Fatalf("after 14 days: events %v, want node_down + host_down", r.events)
|
|
}
|
|
for _, m := range r.msgs {
|
|
if !strings.Contains(m, "household") {
|
|
t.Fatalf("the alarm does not say the household switched it off: %q", m)
|
|
}
|
|
}
|
|
r.check() // once, not every minute
|
|
if len(r.events) != 2 {
|
|
t.Fatalf("repeated: %v", r.events)
|
|
}
|
|
}
|
|
|
|
// Switched back on inside the hold: reports resume → no recovery mail for an outage that never was.
|
|
func TestDeliberateOff_BackOnIsQuiet(t *testing.T) {
|
|
r := newOffRig(t)
|
|
r.offAt(t, 2*time.Hour)
|
|
r.silent(t, 2*time.Hour)
|
|
r.check()
|
|
r.silent(t, 0)
|
|
if _, err := r.st.SetHostDeliberateOff("h1", nil); err != nil { // the new boot's first report
|
|
t.Fatal(err)
|
|
}
|
|
r.check()
|
|
if len(r.events) != 0 {
|
|
t.Fatalf("back on: %v", r.events)
|
|
}
|
|
if r.node.GetState("c1") != "ok" || r.host.GetState("h1") != "ok" {
|
|
t.Fatalf("states node=%q host=%q, want ok", r.node.GetState("c1"), r.host.GetState("h1"))
|
|
}
|
|
}
|
|
|
|
// The 05:00 deadline: Tester 2's shape (R-872) alarms; the same box switched off by the household does not.
|
|
func TestDeliberateOff_DeadlineSkips(t *testing.T) {
|
|
st, _, sc := downBox(t, 5*24*time.Hour)
|
|
if got := deadlineEvents(st, sc); got["expected_backup_missed"] != 1 {
|
|
t.Fatalf("CONTROL: %v — the down box must raise its missed-backup alarm", got)
|
|
}
|
|
st2, _, sc2 := downBox(t, 5*24*time.Hour)
|
|
at := time.Now().UTC().Add(-2 * time.Hour)
|
|
if _, err := st2.SetHostDeliberateOff("h1", &at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := deadlineEvents(st2, sc2); len(got) != 0 {
|
|
t.Fatalf("a switched-off box got deadline alarms: %v", got)
|
|
}
|
|
}
|