package upwatch import ( "context" "errors" "io" "log" "testing" "time" ) // The fake box: containers whose counters the test drives minute by minute, and apps that answer what the test says. type fakeSource struct{ s map[string]*Sample } func (f *fakeSource) Samples(context.Context) ([]Sample, error) { var out []Sample for _, s := range f.s { out = append(out, *s) } return out, nil } func (f *fakeSource) add(container, stack string) *Sample { if f.s == nil { f.s = map[string]*Sample{} } s := &Sample{Container: container, Stack: stack, Started: "t0"} f.s[container] = s return s } type fakeApp struct { stack string rep Report err error } func (a *fakeApp) Stack() string { return a.stack } func (a *fakeApp) Ask(context.Context) (Report, error) { return a.rep, a.err } type event struct { t string d Details } type rig struct { w *Watcher src *fakeSource now time.Time ev []event } func newRig(ex ...*fakeApp) *rig { r := &rig{src: &fakeSource{}, now: time.Date(2026, 10, 23, 22, 0, 0, 0, time.UTC)} m := map[string]Explainer{} for _, e := range ex { m[e.stack] = e } rules := DefaultRules() rules.Cores = 4 r.w = &Watcher{Source: r.src, Explainers: m, Rules: rules, Log: log.New(io.Discard, "", 0), Now: func() time.Time { return r.now }, Emit: func(t, _ string, d Details) { r.ev = append(r.ev, event{t, d}) }} return r } const mbit = 1e6 / 8 // bytes per second at 1 Mbit/s // minute advances the clock one minute; each container sends txMbps and burns cpuPct (of one core) during it. func (r *rig) minute(t *testing.T, step func()) { t.Helper() if step != nil { step() } r.now = r.now.Add(time.Minute) r.w.Tick(context.Background()) } func send(s *Sample, mbps, cpuPct float64) { s.TxBytes += uint64(mbps * mbit * 60) s.CPUNanos += uint64(cpuPct / 100 * 60e9) } func (r *rig) count(t string) int { n := 0 for _, e := range r.ev { if e.t == t { n++ } } return n } // A real torrent seeding at full speed: qBittorrent's own counter explains the bytes (the measured 3–7 % overhead). // RED-PROOF: make the explained test always false → an alarm fires → fails. func TestUpwatch_SeedingQBittorrentIsExplained(t *testing.T) { qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}} r := newRig(qb) c := r.src.add("qbittorrent", "qbittorrent") r.w.Tick(context.Background()) for i := 0; i < 90; i++ { r.minute(t, func() { send(c, 131*1.05, 20) // the container: the app's bytes + 5 % protocol overhead qb.rep.UploadCounter += uint64(131 * mbit * 60) // what qBittorrent counted }) } if len(r.ev) != 0 { t.Fatalf("a seeding qBittorrent raised %v — its own counter explains the upload", r.ev) } } // The fake hacked app: code inside qBittorrent's container floods out while qBittorrent counts nothing. ONE alarm after // 15 minutes, none more while it goes on, ONE clear 15 minutes after it stops. // RED-PROOF: drop `st.upAlarm = true` (no edge) → an alarm every 15 minutes → count != 1 → fails; drop the clear → // no cleared event → fails. func TestUpwatch_HackedQBittorrentAlarmsOnceAndClears(t *testing.T) { qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}} r := newRig(qb) c := r.src.add("qbittorrent", "qbittorrent") r.w.Tick(context.Background()) for i := 1; i <= 14; i++ { r.minute(t, func() { send(c, 900, 30) }) } if len(r.ev) != 0 { t.Fatalf("alarm before 15 minutes: %v", r.ev) } r.minute(t, func() { send(c, 900, 30) }) if r.count("app_unexplained_upload") != 1 { t.Fatalf("after 15 minutes of 900 Mbit/s with qBittorrent counting 0: events %v, want one app_unexplained_upload", r.ev) } d := r.ev[0].d if d.App != "qbittorrent" || d.Kind != "upload" || d.AppMbps != 0 || d.RateMbps < 899 || d.Minutes != 15 || d.Explainer != "qbittorrent" { t.Fatalf("details = %+v", d) } for i := 0; i < 120; i++ { r.minute(t, func() { send(c, 900, 30) }) } if r.count("app_unexplained_upload") != 1 { t.Fatalf("a flood that goes on alarmed %d times — must be ONE mail", r.count("app_unexplained_upload")) } for i := 0; i < 14; i++ { r.minute(t, func() { send(c, 0.01, 1) }) } if r.count("app_unexplained_upload_cleared") != 0 { t.Fatal("cleared before 15 quiet minutes") } r.minute(t, func() { send(c, 0.01, 1) }) if r.count("app_unexplained_upload_cleared") != 1 { t.Fatalf("15 quiet minutes after the flood: events %v, want one clear", r.ev) } } // Plex2Shell's signature: Plex sends a lot with NO viewer. With a viewer, the same bytes are normal use. // RED-PROOF: drop the `winViewers > 0` explanation → the viewer case alarms → fails. func TestUpwatch_PlexWithNoViewerAlarmsWithAViewerDoesNot(t *testing.T) { for _, viewers := range []int{0, 2} { px := &fakeApp{stack: "plex", rep: Report{HasViewerCount: true, Viewers: viewers, HasTranscodeCount: true}} r := newRig(px) c := r.src.add("plex", "plex") r.w.Tick(context.Background()) for i := 0; i < 20; i++ { r.minute(t, func() { send(c, 44, 10) }) } got := r.count("app_unexplained_upload") if viewers == 0 && got != 1 { t.Fatalf("Plex sending 44 Mbit/s with no viewer for 20 min: %d alarm(s), want 1", got) } if viewers > 0 && got != 0 { t.Fatalf("Plex sending to %d viewers alarmed: %v", viewers, r.ev) } } } // An app the box cannot ask: no alarm while it has no history (learning), then only far above its own p95 for an hour. // RED-PROOF: let a no-explainer app use the 15-minute rule → it alarms at 15 → fails. func TestUpwatch_NoExplainerLearnsThenNeedsTenTimesItsP95ForAnHour(t *testing.T) { r := newRig() c := r.src.add("nextcloud", "nextcloud") r.w.Tick(context.Background()) for i := 0; i < 90; i++ { r.minute(t, func() { send(c, 50, 5) }) } if len(r.ev) != 0 { t.Fatalf("an app with no history and no explainer alarmed: %v", r.ev) } r = newRig() c = r.src.add("nextcloud", "nextcloud") r.w.Baseline = func(stack string, cs []string) (float64, bool) { return 2, true } // p95 = 2 Mbit/s → trigger 20 r.w.Tick(context.Background()) for i := 0; i < 59; i++ { r.minute(t, func() { send(c, 50, 5) }) } if len(r.ev) != 0 { t.Fatalf("alarmed before an hour: %v", r.ev) } r.minute(t, func() { send(c, 50, 5) }) if r.count("app_unexplained_upload") != 1 || r.ev[0].d.Explainer != "none" || r.ev[0].d.AppMbps != -1 { t.Fatalf("an hour at 25× its p95: %v", r.ev) } // Below 10× its p95 it is never judged. r = newRig() c = r.src.add("nextcloud", "nextcloud") r.w.Baseline = func(string, []string) (float64, bool) { return 2, true } r.w.Tick(context.Background()) for i := 0; i < 120; i++ { r.minute(t, func() { send(c, 15, 5) }) } if len(r.ev) != 0 { t.Fatalf("15 Mbit/s (7.5× its p95) alarmed: %v", r.ev) } } // The CPU rule: a burner in Plex with no transcode → ONE alarm after 30 minutes; with a transcode, or while the box // itself works on the app (backup, update, restore), none. // RED-PROOF: ignore Transcodes → the transcoding case alarms → fails; ignore Busy → the busy case alarms → fails. func TestUpwatch_CPUBurnerAlarmsTranscodeAndBoxWorkDoNot(t *testing.T) { for _, tc := range []struct { name string transcodes int busy bool want int }{{"burner", 0, false, 1}, {"transcoding", 1, false, 0}, {"box busy", 0, true, 0}} { px := &fakeApp{stack: "plex", rep: Report{HasViewerCount: true, HasTranscodeCount: true, Transcodes: tc.transcodes}} r := newRig(px) r.w.Busy = func(string) bool { return tc.busy } c := r.src.add("plex", "plex") r.w.Tick(context.Background()) for i := 0; i < 29; i++ { r.minute(t, func() { send(c, 0.1, 380) }) // 4 cores: the trigger is max(150, 200) = 200 % } if len(r.ev) != 0 { t.Fatalf("%s: alarm before 30 minutes: %v", tc.name, r.ev) } r.minute(t, func() { send(c, 0.1, 380) }) if got := r.count("app_unexplained_cpu"); got != tc.want { t.Fatalf("%s: %d app_unexplained_cpu, want %d (%v)", tc.name, got, tc.want, r.ev) } if tc.want == 1 { for i := 0; i < 15; i++ { r.minute(t, func() { send(c, 0.1, 5) }) } if r.count("app_unexplained_cpu_cleared") != 1 { t.Fatalf("burner stopped 15 min: %v", r.ev) } } } } // A container that restarts is a new counter, even when the new run's first reading is LARGER than the old run's last // (a busy first minute after a restart) — that minute has no delta. Infrastructure is never judged. // RED-PROOF: drop the Started comparison → the restart minute computes 1 TiB − old → a huge rate → fails. func TestUpwatch_RestartAndInfrastructureAreNotJudged(t *testing.T) { qb := &fakeApp{stack: "qbittorrent", rep: Report{HasUploadCounter: true}} r := newRig(qb) c := r.src.add("qbittorrent", "qbittorrent") ctl := r.src.add("felhom-controller", "felhom-controller") cf := r.src.add("cloudflared", "cloudflared") r.w.Tick(context.Background()) r.minute(t, func() { send(c, 0.1, 1) }) r.minute(t, func() { c.TxBytes, c.Started = 1<<40, "t1" }) // restarted, and its new counter is already 1 TiB if got := r.w.apps["qbittorrent"].lastRate; got > 1 { t.Fatalf("the restart minute was judged at %.0f Mbit/s — a delta across two container runs", got) } r.minute(t, func() { send(c, 0.1, 1) }) if got := r.w.apps["qbittorrent"].lastRate; got < 0.05 || got > 0.2 { t.Fatalf("the minute after a restart: %.3f Mbit/s, want 0.1 (the new run's own delta)", got) } for i := 0; i < 40; i++ { r.minute(t, func() { send(ctl, 900, 400) // the off-site push and the tunnel: never an app alarm send(cf, 900, 10) }) } if len(r.ev) != 0 { t.Fatalf("infrastructure raised %v", r.ev) } if _, judged := r.w.apps["felhom-controller"]; judged { t.Fatal("the controller's own stack was judged as an app") } } // An app that does not answer is „no number": its flood still alarms (the hacked app may refuse its own API). // RED-PROOF: treat an error as "explained" → no alarm → fails. func TestUpwatch_AnAppThatDoesNotAnswerIsNotAnExplanation(t *testing.T) { qb := &fakeApp{stack: "qbittorrent", err: errors.New("connection refused")} r := newRig(qb) c := r.src.add("qbittorrent", "qbittorrent") r.w.Tick(context.Background()) for i := 0; i < 15; i++ { r.minute(t, func() { send(c, 100, 5) }) } if r.count("app_unexplained_upload") != 1 || r.ev[0].d.AppMbps != -1 { t.Fatalf("a silent qBittorrent flooding: %v", r.ev) } } // The trigger follows the measured line: 25 % of it, never under the 5 Mbit/s floor. func TestUpwatch_TriggerFollowsTheLine(t *testing.T) { r := newRig() if got := r.w.UploadTrigger(); got != 5 { t.Fatalf("no line: trigger %v, want the 5 Mbit/s floor", got) } r.w.Line = func() (float64, bool) { return 100, true } if got := r.w.UploadTrigger(); got != 25 { t.Fatalf("100 Mbit/s line: trigger %v, want 25", got) } r.w.Line = func() (float64, bool) { return 8, true } if got := r.w.UploadTrigger(); got != 5 { t.Fatalf("8 Mbit/s line: trigger %v, want the floor", got) } }