hub 0.169.0 (unreleased, not built): the upload watch's four events — allowed, operator-only, per-app cooldown, one readable Watch line in the operator mail
gates / gates (push) Successful in 12m47s

app_unexplained_upload / app_unexplained_cpu (warning) and their _cleared (info), from controller v0.342.0 (08 section 9,
operator ruling 2026-10-23). Red-proved: operator-only, the render line, the allow-list.
This commit is contained in:
2026-10-23 21:26:36 +02:00
parent 5277d859d5
commit af5c005a4e
7 changed files with 140 additions and 2 deletions
+13
View File
@@ -1,3 +1,16 @@
## v0.169.0 (unreleased — committed 2026-10-23 night, NOT built, NOT deployed) — the upload watch's four events
- **`app_unexplained_upload`, `app_unexplained_cpu` (warning) and their `_cleared` (info)** from controller v0.342.0's
upload and CPU watch (`08` §9 [DESIGN — proposed], operator ruling 2026-10-23 19:22): an app sent or burnt far more
than its own numbers explain — the sign of a hacked app (Plex2Shell). In `allowedEventTypes` AND `operatorOnlyEvents`
(the R-388 rule: the household can do nothing about it), per-app cooldown (`perAppCooldownEvents` — two hacked apps
are two alarms; `TestR389_TheAllowListHasExactlyOneMember` widened with its dated reason). The operator mail carries
one `Watch:` line (`renderUnexplained`): the rate, what the app counted (or „no app number to compare"), the minutes,
what was asked, the box's measured line. A clear is info: recorded, not mailed. Tests
`TestUpwatchEvents_OperatorOnlyWithAReadableLine`, `TestUpwatchEventTypesAreAllowedAndOperatorOnly` — red-proved
(dropped from operatorOnlyEvents → the household was mailed; the render call removed → no Watch line; dropped from
the allow-list → fails).
## v0.168.0 (unreleased — built 2026-10-23 night, NOT deployed; the deploy is the operator's yes) — the last plain tokens sealed (R-975, R-1241)
- **R-975 / R-1241's Felhom half.** The Cloudflare tunnel token, the Cloudflare API token and the git token lived in
+7
View File
@@ -2410,6 +2410,13 @@ var allowedEventTypes = map[string]bool{
// definition (a correction, an update, a restore, a settings change) — nothing was written, the app runs on its last
// good compose. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
"app_definition_refused": true,
// The upload and CPU watch (hub v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23): an app sent or burnt
// far more than its own numbers explain — the sign of a hacked app (Plex2Shell) — and the clear when it stopped.
// Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
"app_unexplained_upload": true,
"app_unexplained_upload_cleared": true,
"app_unexplained_cpu": true,
"app_unexplained_cpu_cleared": true,
// Special
"test": true,
}
+20
View File
@@ -0,0 +1,20 @@
package api
import (
"testing"
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
)
// The four upload-watch types are accepted from a box (an unlisted type is a 400 and is dropped) AND operator-only —
// both registers, same commit. RED-PROOF: drop one from allowedEventTypes → fails.
func TestUpwatchEventTypesAreAllowedAndOperatorOnly(t *testing.T) {
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
if !allowedEventTypes[et] {
t.Errorf("%s is not in allowedEventTypes — POST /event would 400 and the alarm would be lost", et)
}
if !notify.IsOperatorOnly(et) {
t.Errorf("%s is not operator-only", et)
}
}
}
+10
View File
@@ -490,6 +490,9 @@ var perAppCooldownEvents = map[string]bool{
// v0.150.0 (R-1007): one app's repair after an unclean boot, no digest behind it — two apps on one boot are two
// records, and a lost photo on the second must not be swallowed by the first's cooldown.
"app_repaired_after_crash": true,
// v0.169.0 (the upload watch): one app's unexplained upload or CPU, no digest — two hacked apps are two alarms.
"app_unexplained_upload": true,
"app_unexplained_cpu": true,
}
// perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The
@@ -892,6 +895,13 @@ var operatorOnlyEvents = map[string]bool{
// of empty files found / repaired / still broken / originals lost. The household learns of a lost photo on its own
// app page (name and date, on its box); THESE counts are the operator's. Registered in the same commit that mints it.
"app_repaired_after_crash": true,
// The upload and CPU watch (v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23 — the R-388 rule): an
// app sending or burning far more than its own numbers explain. The household can do nothing about a hacked app but
// wait for support; the operator decides. The clears too (belt, as for the whole-guest pair). Same commit.
"app_unexplained_upload": true,
"app_unexplained_upload_cleared": true,
"app_unexplained_cpu": true,
"app_unexplained_cpu_cleared": true,
// R-971 (v0.157.0, controller v0.328.0, operator decision D2 = A). The box's catalog allow-list refused an app's
// definition: someone pushed a catalog change asking for container rights the box does not give. The household can
// do nothing; its app page says the app keeps its last good definition. Registered in the same commit that mints it.
@@ -72,7 +72,8 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
// v0.122.0 once more (R-659): a stranded held app is one app's event with no digest.
// v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest.
// v0.150.0 once more (R-1007): an app's repair after an unclean boot is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"}
// v0.169.0 once more (the upload watch): an app's unexplained upload or CPU is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_unexplained_cpu", "app_unexplained_upload", "app_update_held", "app_update_undone"}
ok := len(perAppCooldownEvents) == len(want)
for _, w := range want {
ok = ok && perAppCooldownEvents[w]
@@ -82,7 +83,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
for k := range perAppCooldownEvents {
got = append(got, k)
}
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_unexplained_cpu app_unexplained_upload app_update_held app_update_undone]. Adding a member is the "+
"fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+
"disk sends one digest, not one mail per app. Read the fence before widening this.", got)
}
+38
View File
@@ -70,6 +70,13 @@ Message: %s`, customerID, eventType, severity, now, message)
}
}
// The upload and CPU watch (v0.169.0): what the app sent or burnt, what it said it did, and for how long.
if eventType == "app_unexplained_upload" || eventType == "app_unexplained_cpu" {
if line, ok := renderUnexplained(detailsJSON); ok {
body += "\n" + line
}
}
if detailsJSON != "" && detailsJSON != "{}" {
body += fmt.Sprintf("\nDetails: %s", detailsJSON)
}
@@ -432,6 +439,37 @@ func stackNameOf(detailsJSON string) string {
return d.StackName
}
// renderUnexplained is the upload watch's operator line (v0.169.0, controller upwatch.Details).
func renderUnexplained(detailsJSON string) (string, bool) {
var d struct {
App string `json:"app"`
Kind string `json:"kind"`
Minutes int `json:"minutes"`
RateMbps float64 `json:"rate_mbps"`
AppMbps float64 `json:"app_mbps"`
CPUPercent float64 `json:"cpu_percent"`
Explainer string `json:"explainer"`
Viewers int `json:"viewers"`
Transcodes int `json:"transcodes"`
LineMbps float64 `json:"line_mbps"`
}
if json.Unmarshal([]byte(detailsJSON), &d) != nil || d.App == "" {
return "", false
}
if d.Kind == "cpu" {
return fmt.Sprintf("Watch: %s — %.0f%% CPU (of one core) for %d min; asked: %s, transcodes %d", d.App, d.CPUPercent, d.Minutes, d.Explainer, d.Transcodes), true
}
said := "no app number to compare"
if d.AppMbps >= 0 {
said = fmt.Sprintf("the app counted %.1f Mbit/s", d.AppMbps)
}
line := fmt.Sprintf("Watch: %s — upload %.1f Mbit/s for %d min; %s (asked: %s, viewers %d)", d.App, d.RateMbps, d.Minutes, said, d.Explainer, d.Viewers)
if d.LineMbps > 0 {
line += fmt.Sprintf("; the box's line: at least %.0f Mbit/s", d.LineMbps)
}
return line, true
}
// renderAppRepair is app_repaired_after_crash's operator line (R-1007): the counts, the boot and the duration.
func renderAppRepair(detailsJSON string) (string, bool) {
var d struct {
@@ -0,0 +1,49 @@
package notify
import (
"io"
"log"
"strings"
"testing"
)
// The upload and CPU watch's events (hub v0.169.0, controller v0.342.0) reach the OPERATOR and never the household — even
// one that enabled them by name; the alarm mails once with a readable line; the clear (info) is recorded, not mailed.
// RED-PROOF: drop app_unexplained_upload from operatorOnlyEvents → the household gets a mail → fails; drop the
// renderUnexplained call → the body lacks the Watch line → fails.
func TestUpwatchEvents_OperatorOnlyWithAReadableLine(t *testing.T) {
st := newDispStore(t)
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"app_unexplained_upload", "app_unexplained_cpu", "node_down"}, 6); err != nil {
t.Fatal(err)
}
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
sent := captureSeam(d)
up := `{"app":"qbittorrent","stack_name":"qbittorrent","kind":"upload","minutes":15,"rate_mbps":10136.77,"app_mbps":0,"explainer":"qbittorrent","viewers":0,"transcodes":0,"line_mbps":47}`
d.ProcessEvent("c1", "app_unexplained_upload", "warning", "qbittorrent: ...", up, "controller")
d.ProcessEvent("c1", "app_unexplained_cpu", "warning", "plex: ...", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"app_mbps":-1,"explainer":"plex transcodes","transcodes":0}`, "controller")
if got := mailsFor(*sent, "cust@example.com"); len(got) != 0 {
t.Fatalf("the household got %d mail(s) — a hacked app is the operator's to handle", len(got))
}
if got := len(mailsFor(*sent, "op@felhom.eu")); got != 2 {
t.Fatalf("operator mails = %d, want 2", got)
}
*sent = nil
d.ProcessEvent("c1", "app_unexplained_upload_cleared", "info", "qbittorrent: stopped", `{"app":"qbittorrent","cleared":true,"app_mbps":-1}`, "controller")
if len(*sent) != 0 {
t.Fatalf("a clear (info) mailed %d time(s)", len(*sent))
}
_, body := FormatOperatorEmail("c1", "app_unexplained_upload", "warning", "msg", up)
want := "Watch: qbittorrent — upload 10136.8 Mbit/s for 15 min; the app counted 0.0 Mbit/s (asked: qbittorrent, viewers 0); the box's line: at least 47 Mbit/s"
if !strings.Contains(body, want) {
t.Fatalf("body lacks %q:\n%s", want, body)
}
_, cpu := FormatOperatorEmail("c1", "app_unexplained_cpu", "warning", "msg", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"explainer":"plex transcodes"}`)
if !strings.Contains(cpu, "Watch: plex — 380% CPU (of one core) for 30 min; asked: plex transcodes, transcodes 0") {
t.Fatalf("cpu body:\n%s", cpu)
}
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
if !IsOperatorOnly(et) {
t.Errorf("%s is not operator-only", et)
}
}
}