ab2262c91c
gates / gates (push) Successful in 14s
THE GAP, measured not supposed. On 2026-08-18 ep0's PBS proxy was wedged for
9 h 37 m and the hub emitted NOTHING on the operator channel. Both box
checkers hold their last snapshot and return silently on a failed fetch --
correct for a FILL signal, since a missing reading must never be read as 0%,
but it makes a dead off-site endpoint and a healthy one indistinguishable.
The only mails that morning came from the boxes' own backup failures, and
only because the WEEKLY offsite run happened to land inside the window. Two
days earlier nothing would have fired at all.
REACHABILITY is now a second, independent signal on both checkers:
consecutive failed fetch windows, reported past a default 3 windows
(~30-45 min) as pbsdr_box_unreachable / offsite_box_unreachable (warning) on
the customer-less pbsdr-box / pool-box scopes, each with a paired *_recovered
all-clear. Tunable via alerting.box_unreachable_windows (0/invalid -> 3).
THE FILL LOGIC IS UNTOUCHED. No threshold, throttle, band or escalate-once
behaviour changed; a degraded read still drives no transition.
Three decisions a later reader would otherwise "fix" back, so each is
argued in-code:
- the unreachable event REPEATS rather than escalating once. The band shape
would give exactly ONE mail at ~minute 30 of a nine-hour outage, and one
mail is missable. It leans on the dispatcher's 1 h operator cooldown to
become an hourly "still blind" heartbeat.
- ErrUsageUnsupported is NOT blindness: an old ep0 answers "no such op",
which means we reached it. Counting it would alert for days on a healthy
pre-update endpoint.
- born-blind is reported: the counter is not gated on having a snapshot, so
a hub restarted INTO an outage still speaks. last_ok is OMITTED rather
than zero-valued -- a fabricated timestamp reads as "it was fine until
then".
Both recoveries are severity "info" and severityNotifies drops "info", so
they are registered in recoveredPairedDownTypes or the operator hears that
the tier broke and never that it healed. A cross-package test drives
ProcessEvent and asserts an actual operator MAIL, not a map entry -- a green
checker test proves nothing about the seam (agent v0.91.0 shipped fully green
with SetAuthSink never called).
Tests: box_reachability_test.go (Scenarios A-F) + dispatcher_box_reachability
_test.go (wiring). Three red-proofs run and reverted, each seen failing with a
message naming the right cause: threshold 3->1, the sentinel counter guard,
the pairing entry.
Register: R-339 filed and marked SHIPPED (PROVEN-LIVE still owed -- no real or
constructed outage has exercised the emit path, and one cannot be manufactured
against Tier-2 ep0). R-340 filed: the reachability read rides ep0's LOCAL API
daemon, which the incident explicitly cleared, so this check would have shown
GREEN for all 9 h 37 m -- the honest boundary, recorded rather than glossed.
R-336's next-step corrected: pvestatd's interval is NOT tunable (Proxmox staff
have said so); the only lever is disabling the storage entry, which collides
with the agent's consume-the-one-time-secret path. Doc-only, no agent code
touched.
161 lines
5.4 KiB
Go
161 lines
5.4 KiB
Go
package monitor
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
|
|
)
|
|
|
|
// fakeUsage is a usageReader for tests: it returns a scripted usage/err and counts calls.
|
|
type fakeUsage struct {
|
|
mu sync.Mutex
|
|
usage tenantsync.BoxUsage
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeUsage) Usage(_ context.Context) (tenantsync.BoxUsage, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls++
|
|
return f.usage, f.err
|
|
}
|
|
func (f *fakeUsage) set(u tenantsync.BoxUsage, err error) {
|
|
f.mu.Lock()
|
|
f.usage, f.err = u, err
|
|
f.mu.Unlock()
|
|
}
|
|
func (f *fakeUsage) nCalls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.calls }
|
|
|
|
func usageBytes(totalGiB, usedGiB int64) tenantsync.BoxUsage {
|
|
return tenantsync.BoxUsage{Total: totalGiB * gib, Used: usedGiB * gib, Avail: (totalGiB - usedGiB) * gib}
|
|
}
|
|
|
|
// C1 — throttle: ≤1 ep0 usage call per 15-min window across an hour of 60 s sweeps (≈4, not ≈60).
|
|
func TestPBSDRBox_Throttle(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(40, 8)}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, noEvent, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
base := time.Now().UTC()
|
|
for i := 0; i < 60; i++ {
|
|
cur = base.Add(time.Duration(i) * 60 * time.Second)
|
|
c.Check()
|
|
}
|
|
if f.nCalls() < 3 || f.nCalls() > 5 {
|
|
t.Fatalf("usage op called %d times over an hour of 60 s sweeps, want ≈4 (throttled to 15 min)", f.nCalls())
|
|
}
|
|
snap, ok := c.Snapshot()
|
|
if !ok || snap.State != PBSStateOK || snap.CapacityBytes != 40*gib {
|
|
t.Fatalf("snapshot wrong: %+v ok=%v", snap, ok)
|
|
}
|
|
}
|
|
|
|
// C2/C3 — fill bands: escalation-only, in-band no re-emit, recovery re-arm, pbsdr-box scope.
|
|
// Red-proof (esc): remove the escalation-only guard → the C3 same-band case re-emits → fails.
|
|
func TestPBSDRBox_FillBands(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(1000, 750)} // 75%
|
|
ev := &capturedBox{}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
base := time.Now().UTC()
|
|
cur = base
|
|
step := func(usedGiB int64) {
|
|
cur = cur.Add(16 * time.Minute)
|
|
f.set(usageBytes(1000, usedGiB), nil)
|
|
c.Check()
|
|
}
|
|
|
|
c.Check() // 75% → no emit
|
|
if len(ev.typ) != 0 {
|
|
t.Fatalf("75%% must not emit, got %v", ev.typ)
|
|
}
|
|
step(820) // 82% → one warning
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 || ev.sev[0] != "warning" {
|
|
t.Fatalf("82%% must warn once, got typ=%v sev=%v", ev.typ, ev.sev)
|
|
}
|
|
step(850) // 85% same band → no re-emit
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("same-band must NOT re-emit, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
step(920) // 92% → critical
|
|
if count(ev.typ, "pbsdr_box_fill") != 2 || ev.sev[len(ev.sev)-1] != "critical" {
|
|
t.Fatalf("92%% must escalate to critical, got typ=%v sev=%v", ev.typ, ev.sev)
|
|
}
|
|
step(700) // 70% → recovery re-arm (silent)
|
|
if c.FillState() != bandOK {
|
|
t.Fatalf("recovery must re-arm to ok, got %s", c.FillState())
|
|
}
|
|
if count(ev.typ, "pbsdr_box_fill") != 2 {
|
|
t.Fatalf("recovery must be silent, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
step(920) // re-breach → emits again
|
|
if count(ev.typ, "pbsdr_box_fill") != 3 {
|
|
t.Fatalf("re-breach after recovery must emit again, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
for _, cust := range ev.cust {
|
|
if cust != "pbsdr-box" {
|
|
t.Fatalf("every emit must carry the pbsdr-box scope, got %q", cust)
|
|
}
|
|
}
|
|
}
|
|
|
|
// C4 — usage-unsupported (ErrUsageUnsupported) → the distinct "unavailable" state, NOT degraded, NO
|
|
// alert, NO band transition. Red-proof (unavail-band): let unavailable drive a band → alert fires → fails.
|
|
func TestPBSDRBox_Unavailable(t *testing.T) {
|
|
f := &fakeUsage{err: tenantsync.ErrUsageUnsupported}
|
|
ev := &capturedBox{}
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.Check()
|
|
snap, ok := c.Snapshot()
|
|
if !ok || snap.State != PBSStateUnavailable {
|
|
t.Fatalf("ErrUsageUnsupported must yield the 'unavailable' state, got %+v", snap)
|
|
}
|
|
if len(ev.typ) != 0 {
|
|
t.Fatalf("unavailable must NOT alert, got %v", ev.typ)
|
|
}
|
|
if c.FillState() != "unknown" {
|
|
t.Fatalf("unavailable must not set a fill band, got %s", c.FillState())
|
|
}
|
|
}
|
|
|
|
// C5 — exec error/timeout → last snapshot kept, marked degraded, NO band transition (missing ≠ 0%).
|
|
func TestPBSDRBox_DegradedKeepsLast(t *testing.T) {
|
|
f := &fakeUsage{usage: usageBytes(1000, 920)} // 92% critical
|
|
ev := &capturedBox{}
|
|
var cur time.Time
|
|
c := NewPBSDRBoxChecker(f, 80, 90, 0, ev.fn, quietLog())
|
|
c.now = func() time.Time { return cur }
|
|
cur = time.Now().UTC()
|
|
|
|
c.Check() // establish 92% critical + emit
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("setup: want one critical emit, got %v", ev.typ)
|
|
}
|
|
before, _ := c.Snapshot()
|
|
|
|
f.set(tenantsync.BoxUsage{}, fmt.Errorf("ssh timeout"))
|
|
cur = cur.Add(16 * time.Minute)
|
|
c.Check()
|
|
|
|
after, ok := c.Snapshot()
|
|
if !ok || after.State != PBSStateDegraded {
|
|
t.Fatalf("a failed poll must mark degraded (keeping last), got %+v", after)
|
|
}
|
|
if after.UsedBytes != before.UsedBytes || after.CapacityBytes != before.CapacityBytes {
|
|
t.Fatalf("a failed poll must keep the last-known values (before %d/%d, after %d/%d)",
|
|
before.UsedBytes, before.CapacityBytes, after.UsedBytes, after.CapacityBytes)
|
|
}
|
|
if c.FillState() != bandCritical {
|
|
t.Fatalf("a failed poll must NOT transition the band, got %s", c.FillState())
|
|
}
|
|
if count(ev.typ, "pbsdr_box_fill") != 1 {
|
|
t.Fatalf("a failed poll must not emit, got %d", count(ev.typ, "pbsdr_box_fill"))
|
|
}
|
|
}
|