admin
a9c1997fbc
gates / gates (push) Successful in 1m17s
internal/upwatch: exact per-container counters from the Docker Engine API, per-app judging (upload >= max(5 Mbit/s, 25% of the line) for 15 min; CPU >= max(150%, 50% x cores) for 30 min), explainers qBittorrent / Transmission / Plex, app_unexplained_upload / app_unexplained_cpu + _cleared (hub 0.169.0). The off-site push is sampled once a second for the line (a lower bound). qBittorrent limit = 50% of the line, the household's limit always wins. 20 red-proofs. Spike + design: felhom.eu audits/SPIKE-upload-watch-2026-10-23.md, architecture/08 section 9. Not released.
313 lines
11 KiB
Go
313 lines
11 KiB
Go
package upwatch
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"io"
|
||
"log"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// The fake box: containers whose counters the test drives minute by minute, and apps that answer what the test says.
|
||
|
||
type fakeSource struct{ s map[string]*Sample }
|
||
|
||
func (f *fakeSource) Samples(context.Context) ([]Sample, error) {
|
||
var out []Sample
|
||
for _, s := range f.s {
|
||
out = append(out, *s)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (f *fakeSource) add(container, stack string) *Sample {
|
||
if f.s == nil {
|
||
f.s = map[string]*Sample{}
|
||
}
|
||
s := &Sample{Container: container, Stack: stack, Started: "t0"}
|
||
f.s[container] = s
|
||
return s
|
||
}
|
||
|
||
type fakeApp struct {
|
||
stack string
|
||
rep Report
|
||
err error
|
||
}
|
||
|
||
func (a *fakeApp) Stack() string { return a.stack }
|
||
func (a *fakeApp) Ask(context.Context) (Report, error) { return a.rep, a.err }
|
||
|
||
type event struct {
|
||
t string
|
||
d Details
|
||
}
|
||
|
||
type rig struct {
|
||
w *Watcher
|
||
src *fakeSource
|
||
now time.Time
|
||
ev []event
|
||
}
|
||
|
||
func newRig(ex ...*fakeApp) *rig {
|
||
r := &rig{src: &fakeSource{}, now: time.Date(2026, 10, 23, 22, 0, 0, 0, time.UTC)}
|
||
m := map[string]Explainer{}
|
||
for _, e := range ex {
|
||
m[e.stack] = e
|
||
}
|
||
rules := DefaultRules()
|
||
rules.Cores = 4
|
||
r.w = &Watcher{Source: r.src, Explainers: m, Rules: rules, Log: log.New(io.Discard, "", 0),
|
||
Now: func() time.Time { return r.now },
|
||
Emit: func(t, _ string, d Details) { r.ev = append(r.ev, event{t, d}) }}
|
||
return r
|
||
}
|
||
|
||
const mbit = 1e6 / 8 // bytes per second at 1 Mbit/s
|
||
|
||
// minute advances the clock one minute; each container sends txMbps and burns cpuPct (of one core) during it.
|
||
func (r *rig) minute(t *testing.T, step func()) {
|
||
t.Helper()
|
||
if step != nil {
|
||
step()
|
||
}
|
||
r.now = r.now.Add(time.Minute)
|
||
r.w.Tick(context.Background())
|
||
}
|
||
|
||
func send(s *Sample, mbps, cpuPct float64) {
|
||
s.TxBytes += uint64(mbps * mbit * 60)
|
||
s.CPUNanos += uint64(cpuPct / 100 * 60e9)
|
||
}
|
||
|
||
func (r *rig) count(t string) int {
|
||
n := 0
|
||
for _, e := range r.ev {
|
||
if e.t == t {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
// A real torrent seeding at full speed: qBittorrent's own counter explains the bytes (the measured 3–7 % overhead).
|
||
// RED-PROOF: make the explained test always false → an alarm fires → fails.
|
||
func TestUpwatch_SeedingQBittorrentIsExplained(t *testing.T) {
|
||
qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}}
|
||
r := newRig(qb)
|
||
c := r.src.add("qbittorrent", "qbittorrent")
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 90; i++ {
|
||
r.minute(t, func() {
|
||
send(c, 131*1.05, 20) // the container: the app's bytes + 5 % protocol overhead
|
||
qb.rep.UploadCounter += uint64(131 * mbit * 60) // what qBittorrent counted
|
||
})
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("a seeding qBittorrent raised %v — its own counter explains the upload", r.ev)
|
||
}
|
||
}
|
||
|
||
// The fake hacked app: code inside qBittorrent's container floods out while qBittorrent counts nothing. ONE alarm after
|
||
// 15 minutes, none more while it goes on, ONE clear 15 minutes after it stops.
|
||
// RED-PROOF: drop `st.upAlarm = true` (no edge) → an alarm every 15 minutes → count != 1 → fails; drop the clear →
|
||
// no cleared event → fails.
|
||
func TestUpwatch_HackedQBittorrentAlarmsOnceAndClears(t *testing.T) {
|
||
qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}}
|
||
r := newRig(qb)
|
||
c := r.src.add("qbittorrent", "qbittorrent")
|
||
r.w.Tick(context.Background())
|
||
for i := 1; i <= 14; i++ {
|
||
r.minute(t, func() { send(c, 900, 30) })
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("alarm before 15 minutes: %v", r.ev)
|
||
}
|
||
r.minute(t, func() { send(c, 900, 30) })
|
||
if r.count("app_unexplained_upload") != 1 {
|
||
t.Fatalf("after 15 minutes of 900 Mbit/s with qBittorrent counting 0: events %v, want one app_unexplained_upload", r.ev)
|
||
}
|
||
d := r.ev[0].d
|
||
if d.App != "qbittorrent" || d.Kind != "upload" || d.AppMbps != 0 || d.RateMbps < 899 || d.Minutes != 15 || d.Explainer != "qbittorrent" {
|
||
t.Fatalf("details = %+v", d)
|
||
}
|
||
for i := 0; i < 120; i++ {
|
||
r.minute(t, func() { send(c, 900, 30) })
|
||
}
|
||
if r.count("app_unexplained_upload") != 1 {
|
||
t.Fatalf("a flood that goes on alarmed %d times — must be ONE mail", r.count("app_unexplained_upload"))
|
||
}
|
||
for i := 0; i < 14; i++ {
|
||
r.minute(t, func() { send(c, 0.01, 1) })
|
||
}
|
||
if r.count("app_unexplained_upload_cleared") != 0 {
|
||
t.Fatal("cleared before 15 quiet minutes")
|
||
}
|
||
r.minute(t, func() { send(c, 0.01, 1) })
|
||
if r.count("app_unexplained_upload_cleared") != 1 {
|
||
t.Fatalf("15 quiet minutes after the flood: events %v, want one clear", r.ev)
|
||
}
|
||
}
|
||
|
||
// Plex2Shell's signature: Plex sends a lot with NO viewer. With a viewer, the same bytes are normal use.
|
||
// RED-PROOF: drop the `winViewers > 0` explanation → the viewer case alarms → fails.
|
||
func TestUpwatch_PlexWithNoViewerAlarmsWithAViewerDoesNot(t *testing.T) {
|
||
for _, viewers := range []int{0, 2} {
|
||
px := &fakeApp{stack: "plex", rep: Report{HasViewerCount: true, Viewers: viewers, HasTranscodeCount: true}}
|
||
r := newRig(px)
|
||
c := r.src.add("plex", "plex")
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 20; i++ {
|
||
r.minute(t, func() { send(c, 44, 10) })
|
||
}
|
||
got := r.count("app_unexplained_upload")
|
||
if viewers == 0 && got != 1 {
|
||
t.Fatalf("Plex sending 44 Mbit/s with no viewer for 20 min: %d alarm(s), want 1", got)
|
||
}
|
||
if viewers > 0 && got != 0 {
|
||
t.Fatalf("Plex sending to %d viewers alarmed: %v", viewers, r.ev)
|
||
}
|
||
}
|
||
}
|
||
|
||
// An app the box cannot ask: no alarm while it has no history (learning), then only far above its own p95 for an hour.
|
||
// RED-PROOF: let a no-explainer app use the 15-minute rule → it alarms at 15 → fails.
|
||
func TestUpwatch_NoExplainerLearnsThenNeedsTenTimesItsP95ForAnHour(t *testing.T) {
|
||
r := newRig()
|
||
c := r.src.add("nextcloud", "nextcloud")
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 90; i++ {
|
||
r.minute(t, func() { send(c, 50, 5) })
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("an app with no history and no explainer alarmed: %v", r.ev)
|
||
}
|
||
r = newRig()
|
||
c = r.src.add("nextcloud", "nextcloud")
|
||
r.w.Baseline = func(stack string, cs []string) (float64, bool) { return 2, true } // p95 = 2 Mbit/s → trigger 20
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 59; i++ {
|
||
r.minute(t, func() { send(c, 50, 5) })
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("alarmed before an hour: %v", r.ev)
|
||
}
|
||
r.minute(t, func() { send(c, 50, 5) })
|
||
if r.count("app_unexplained_upload") != 1 || r.ev[0].d.Explainer != "none" || r.ev[0].d.AppMbps != -1 {
|
||
t.Fatalf("an hour at 25× its p95: %v", r.ev)
|
||
}
|
||
// Below 10× its p95 it is never judged.
|
||
r = newRig()
|
||
c = r.src.add("nextcloud", "nextcloud")
|
||
r.w.Baseline = func(string, []string) (float64, bool) { return 2, true }
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 120; i++ {
|
||
r.minute(t, func() { send(c, 15, 5) })
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("15 Mbit/s (7.5× its p95) alarmed: %v", r.ev)
|
||
}
|
||
}
|
||
|
||
// The CPU rule: a burner in Plex with no transcode → ONE alarm after 30 minutes; with a transcode, or while the box
|
||
// itself works on the app (backup, update, restore), none.
|
||
// RED-PROOF: ignore Transcodes → the transcoding case alarms → fails; ignore Busy → the busy case alarms → fails.
|
||
func TestUpwatch_CPUBurnerAlarmsTranscodeAndBoxWorkDoNot(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
name string
|
||
transcodes int
|
||
busy bool
|
||
want int
|
||
}{{"burner", 0, false, 1}, {"transcoding", 1, false, 0}, {"box busy", 0, true, 0}} {
|
||
px := &fakeApp{stack: "plex", rep: Report{HasViewerCount: true, HasTranscodeCount: true, Transcodes: tc.transcodes}}
|
||
r := newRig(px)
|
||
r.w.Busy = func(string) bool { return tc.busy }
|
||
c := r.src.add("plex", "plex")
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 29; i++ {
|
||
r.minute(t, func() { send(c, 0.1, 380) }) // 4 cores: the trigger is max(150, 200) = 200 %
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("%s: alarm before 30 minutes: %v", tc.name, r.ev)
|
||
}
|
||
r.minute(t, func() { send(c, 0.1, 380) })
|
||
if got := r.count("app_unexplained_cpu"); got != tc.want {
|
||
t.Fatalf("%s: %d app_unexplained_cpu, want %d (%v)", tc.name, got, tc.want, r.ev)
|
||
}
|
||
if tc.want == 1 {
|
||
for i := 0; i < 15; i++ {
|
||
r.minute(t, func() { send(c, 0.1, 5) })
|
||
}
|
||
if r.count("app_unexplained_cpu_cleared") != 1 {
|
||
t.Fatalf("burner stopped 15 min: %v", r.ev)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// A container that restarts is a new counter, even when the new run's first reading is LARGER than the old run's last
|
||
// (a busy first minute after a restart) — that minute has no delta. Infrastructure is never judged.
|
||
// RED-PROOF: drop the Started comparison → the restart minute computes 1 TiB − old → a huge rate → fails.
|
||
func TestUpwatch_RestartAndInfrastructureAreNotJudged(t *testing.T) {
|
||
qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}}
|
||
r := newRig(qb)
|
||
c := r.src.add("qbittorrent", "qbittorrent")
|
||
ctl := r.src.add("felhom-controller", "felhom-controller")
|
||
cf := r.src.add("cloudflared", "cloudflared")
|
||
r.w.Tick(context.Background())
|
||
r.minute(t, func() { send(c, 0.1, 1) })
|
||
r.minute(t, func() { c.TxBytes, c.Started = 1<<40, "t1" }) // restarted, and its new counter is already 1 TiB
|
||
if got := r.w.apps["qbittorrent"].lastRate; got > 1 {
|
||
t.Fatalf("the restart minute was judged at %.0f Mbit/s — a delta across two container runs", got)
|
||
}
|
||
r.minute(t, func() { send(c, 0.1, 1) })
|
||
if got := r.w.apps["qbittorrent"].lastRate; got < 0.05 || got > 0.2 {
|
||
t.Fatalf("the minute after a restart: %.3f Mbit/s, want 0.1 (the new run's own delta)", got)
|
||
}
|
||
for i := 0; i < 40; i++ {
|
||
r.minute(t, func() {
|
||
send(ctl, 900, 400) // the off-site push and the tunnel: never an app alarm
|
||
send(cf, 900, 10)
|
||
})
|
||
}
|
||
if len(r.ev) != 0 {
|
||
t.Fatalf("infrastructure raised %v", r.ev)
|
||
}
|
||
if _, judged := r.w.apps["felhom-controller"]; judged {
|
||
t.Fatal("the controller's own stack was judged as an app")
|
||
}
|
||
}
|
||
|
||
// An app that does not answer is „no number": its flood still alarms (the hacked app may refuse its own API).
|
||
// RED-PROOF: treat an error as "explained" → no alarm → fails.
|
||
func TestUpwatch_AnAppThatDoesNotAnswerIsNotAnExplanation(t *testing.T) {
|
||
qb := &fakeApp{stack: "qbittorrent", err: errors.New("connection refused")}
|
||
r := newRig(qb)
|
||
c := r.src.add("qbittorrent", "qbittorrent")
|
||
r.w.Tick(context.Background())
|
||
for i := 0; i < 15; i++ {
|
||
r.minute(t, func() { send(c, 100, 5) })
|
||
}
|
||
if r.count("app_unexplained_upload") != 1 || r.ev[0].d.AppMbps != -1 {
|
||
t.Fatalf("a silent qBittorrent flooding: %v", r.ev)
|
||
}
|
||
}
|
||
|
||
// The trigger follows the measured line: 25 % of it, never under the 5 Mbit/s floor.
|
||
func TestUpwatch_TriggerFollowsTheLine(t *testing.T) {
|
||
r := newRig()
|
||
if got := r.w.UploadTrigger(); got != 5 {
|
||
t.Fatalf("no line: trigger %v, want the 5 Mbit/s floor", got)
|
||
}
|
||
r.w.Line = func() (float64, bool) { return 100, true }
|
||
if got := r.w.UploadTrigger(); got != 25 {
|
||
t.Fatalf("100 Mbit/s line: trigger %v, want 25", got)
|
||
}
|
||
r.w.Line = func() (float64, bool) { return 8, true }
|
||
if got := r.w.UploadTrigger(); got != 5 {
|
||
t.Fatalf("8 Mbit/s line: trigger %v, want the floor", got)
|
||
}
|
||
}
|