hub 0.169.0 (unreleased, not built): the upload watch's four events — allowed, operator-only, per-app cooldown, one readable Watch line in the operator mail
gates / gates (push) Successful in 12m47s

app_unexplained_upload / app_unexplained_cpu (warning) and their _cleared (info), from controller v0.342.0 (08 section 9,
operator ruling 2026-10-23). Red-proved: operator-only, the render line, the allow-list.
This commit is contained in:
admin committed 2026-10-23 21:26:36 +02:00
1 parent 5277d859d5
commit af5c005a4e
7 files changed
+140 -2

No files matched your search

+13
View File
@@ -1,3 +1,16 @@
## v0.169.0 (unreleased — committed 2026-10-23 night, NOT built, NOT deployed) — the upload watch's four events
- **`app_unexplained_upload`, `app_unexplained_cpu` (warning) and their `_cleared` (info)** from controller v0.342.0's
upload and CPU watch (`08` §9 [DESIGN — proposed], operator ruling 2026-10-23 19:22): an app sent or burnt far more
than its own numbers explain — the sign of a hacked app (Plex2Shell). In `allowedEventTypes` AND `operatorOnlyEvents`
(the R-388 rule: the household can do nothing about it), per-app cooldown (`perAppCooldownEvents` — two hacked apps
are two alarms; `TestR389_TheAllowListHasExactlyOneMember` widened with its dated reason). The operator mail carries
one `Watch:` line (`renderUnexplained`): the rate, what the app counted (or „no app number to compare"), the minutes,
what was asked, the box's measured line. A clear is info: recorded, not mailed. Tests
`TestUpwatchEvents_OperatorOnlyWithAReadableLine`, `TestUpwatchEventTypesAreAllowedAndOperatorOnly` — red-proved
(dropped from operatorOnlyEvents → the household was mailed; the render call removed → no Watch line; dropped from
the allow-list → fails).
## v0.168.0 (unreleased — built 2026-10-23 night, NOT deployed; the deploy is the operator's yes) — the last plain tokens sealed (R-975, R-1241) ## v0.168.0 (unreleased — built 2026-10-23 night, NOT deployed; the deploy is the operator's yes) — the last plain tokens sealed (R-975, R-1241)
- **R-975 / R-1241's Felhom half.** The Cloudflare tunnel token, the Cloudflare API token and the git token lived in - **R-975 / R-1241's Felhom half.** The Cloudflare tunnel token, the Cloudflare API token and the git token lived in
+7
View File
@@ -2410,6 +2410,13 @@ var allowedEventTypes = map[string]bool{
// definition (a correction, an update, a restore, a settings change) — nothing was written, the app runs on its last // definition (a correction, an update, a restore, a settings change) — nothing was written, the app runs on its last
// good compose. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit. // good compose. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
"app_definition_refused": true, "app_definition_refused": true,
// The upload and CPU watch (hub v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23): an app sent or burnt
// far more than its own numbers explain — the sign of a hacked app (Plex2Shell) — and the clear when it stopped.
// Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
"app_unexplained_upload": true,
"app_unexplained_upload_cleared": true,
"app_unexplained_cpu": true,
"app_unexplained_cpu_cleared": true,
// Special // Special
"test": true, "test": true,
} }
+20
View File
@@ -0,0 +1,20 @@
package api
import (
"testing"
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
)
// The four upload-watch types are accepted from a box (an unlisted type is a 400 and is dropped) AND operator-only —
// both registers, same commit. RED-PROOF: drop one from allowedEventTypes → fails.
func TestUpwatchEventTypesAreAllowedAndOperatorOnly(t *testing.T) {
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
if !allowedEventTypes[et] {
t.Errorf("%s is not in allowedEventTypes — POST /event would 400 and the alarm would be lost", et)
}
if !notify.IsOperatorOnly(et) {
t.Errorf("%s is not operator-only", et)
}
}
}
+10
View File
@@ -490,6 +490,9 @@ var perAppCooldownEvents = map[string]bool{
// v0.150.0 (R-1007): one app's repair after an unclean boot, no digest behind it — two apps on one boot are two // v0.150.0 (R-1007): one app's repair after an unclean boot, no digest behind it — two apps on one boot are two
// records, and a lost photo on the second must not be swallowed by the first's cooldown. // records, and a lost photo on the second must not be swallowed by the first's cooldown.
"app_repaired_after_crash": true, "app_repaired_after_crash": true,
// v0.169.0 (the upload watch): one app's unexplained upload or CPU, no digest — two hacked apps are two alarms.
"app_unexplained_upload": true,
"app_unexplained_cpu": true,
} }
// perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The // perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The
@@ -892,6 +895,13 @@ var operatorOnlyEvents = map[string]bool{
// of empty files found / repaired / still broken / originals lost. The household learns of a lost photo on its own // of empty files found / repaired / still broken / originals lost. The household learns of a lost photo on its own
// app page (name and date, on its box); THESE counts are the operator's. Registered in the same commit that mints it. // app page (name and date, on its box); THESE counts are the operator's. Registered in the same commit that mints it.
"app_repaired_after_crash": true, "app_repaired_after_crash": true,
// The upload and CPU watch (v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23 — the R-388 rule): an
// app sending or burning far more than its own numbers explain. The household can do nothing about a hacked app but
// wait for support; the operator decides. The clears too (belt, as for the whole-guest pair). Same commit.
"app_unexplained_upload": true,
"app_unexplained_upload_cleared": true,
"app_unexplained_cpu": true,
"app_unexplained_cpu_cleared": true,
// R-971 (v0.157.0, controller v0.328.0, operator decision D2 = A). The box's catalog allow-list refused an app's // R-971 (v0.157.0, controller v0.328.0, operator decision D2 = A). The box's catalog allow-list refused an app's
// definition: someone pushed a catalog change asking for container rights the box does not give. The household can // definition: someone pushed a catalog change asking for container rights the box does not give. The household can
// do nothing; its app page says the app keeps its last good definition. Registered in the same commit that mints it. // do nothing; its app page says the app keeps its last good definition. Registered in the same commit that mints it.
@@ -72,7 +72,8 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
// v0.122.0 once more (R-659): a stranded held app is one app's event with no digest. // v0.122.0 once more (R-659): a stranded held app is one app's event with no digest.
// v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest. // v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest.
// v0.150.0 once more (R-1007): an app's repair after an unclean boot is one app's event with no digest. // v0.150.0 once more (R-1007): an app's repair after an unclean boot is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"} // v0.169.0 once more (the upload watch): an app's unexplained upload or CPU is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_unexplained_cpu", "app_unexplained_upload", "app_update_held", "app_update_undone"}
ok := len(perAppCooldownEvents) == len(want) ok := len(perAppCooldownEvents) == len(want)
for _, w := range want { for _, w := range want {
ok = ok && perAppCooldownEvents[w] ok = ok && perAppCooldownEvents[w]
@@ -82,7 +83,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
for k := range perAppCooldownEvents { for k := range perAppCooldownEvents {
got = append(got, k) got = append(got, k)
} }
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+ t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_unexplained_cpu app_unexplained_upload app_update_held app_update_undone]. Adding a member is the "+
"fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+ "fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+
"disk sends one digest, not one mail per app. Read the fence before widening this.", got) "disk sends one digest, not one mail per app. Read the fence before widening this.", got)
} }
+38
View File
@@ -70,6 +70,13 @@ Message: %s`, customerID, eventType, severity, now, message)
} }
} }
// The upload and CPU watch (v0.169.0): what the app sent or burnt, what it said it did, and for how long.
if eventType == "app_unexplained_upload" || eventType == "app_unexplained_cpu" {
if line, ok := renderUnexplained(detailsJSON); ok {
body += "\n" + line
}
}
if detailsJSON != "" && detailsJSON != "{}" { if detailsJSON != "" && detailsJSON != "{}" {
body += fmt.Sprintf("\nDetails: %s", detailsJSON) body += fmt.Sprintf("\nDetails: %s", detailsJSON)
} }
@@ -432,6 +439,37 @@ func stackNameOf(detailsJSON string) string {
return d.StackName return d.StackName
} }
// renderUnexplained is the upload watch's operator line (v0.169.0, controller upwatch.Details).
func renderUnexplained(detailsJSON string) (string, bool) {
var d struct {
App string `json:"app"`
Kind string `json:"kind"`
Minutes int `json:"minutes"`
RateMbps float64 `json:"rate_mbps"`
AppMbps float64 `json:"app_mbps"`
CPUPercent float64 `json:"cpu_percent"`
Explainer string `json:"explainer"`
Viewers int `json:"viewers"`
Transcodes int `json:"transcodes"`
LineMbps float64 `json:"line_mbps"`
}
if json.Unmarshal([]byte(detailsJSON), &d) != nil || d.App == "" {
return "", false
}
if d.Kind == "cpu" {
return fmt.Sprintf("Watch: %s — %.0f%% CPU (of one core) for %d min; asked: %s, transcodes %d", d.App, d.CPUPercent, d.Minutes, d.Explainer, d.Transcodes), true
}
said := "no app number to compare"
if d.AppMbps >= 0 {
said = fmt.Sprintf("the app counted %.1f Mbit/s", d.AppMbps)
}
line := fmt.Sprintf("Watch: %s — upload %.1f Mbit/s for %d min; %s (asked: %s, viewers %d)", d.App, d.RateMbps, d.Minutes, said, d.Explainer, d.Viewers)
if d.LineMbps > 0 {
line += fmt.Sprintf("; the box's line: at least %.0f Mbit/s", d.LineMbps)
}
return line, true
}
// renderAppRepair is app_repaired_after_crash's operator line (R-1007): the counts, the boot and the duration. // renderAppRepair is app_repaired_after_crash's operator line (R-1007): the counts, the boot and the duration.
func renderAppRepair(detailsJSON string) (string, bool) { func renderAppRepair(detailsJSON string) (string, bool) {
var d struct { var d struct {
@@ -0,0 +1,49 @@
package notify
import (
"io"
"log"
"strings"
"testing"
)
// The upload and CPU watch's events (hub v0.169.0, controller v0.342.0) reach the OPERATOR and never the household — even
// one that enabled them by name; the alarm mails once with a readable line; the clear (info) is recorded, not mailed.
// RED-PROOF: drop app_unexplained_upload from operatorOnlyEvents → the household gets a mail → fails; drop the
// renderUnexplained call → the body lacks the Watch line → fails.
func TestUpwatchEvents_OperatorOnlyWithAReadableLine(t *testing.T) {
st := newDispStore(t)
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"app_unexplained_upload", "app_unexplained_cpu", "node_down"}, 6); err != nil {
t.Fatal(err)
}
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
sent := captureSeam(d)
up := `{"app":"qbittorrent","stack_name":"qbittorrent","kind":"upload","minutes":15,"rate_mbps":10136.77,"app_mbps":0,"explainer":"qbittorrent","viewers":0,"transcodes":0,"line_mbps":47}`
d.ProcessEvent("c1", "app_unexplained_upload", "warning", "qbittorrent: ...", up, "controller")
d.ProcessEvent("c1", "app_unexplained_cpu", "warning", "plex: ...", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"app_mbps":-1,"explainer":"plex transcodes","transcodes":0}`, "controller")
if got := mailsFor(*sent, "cust@example.com"); len(got) != 0 {
t.Fatalf("the household got %d mail(s) — a hacked app is the operator's to handle", len(got))
}
if got := len(mailsFor(*sent, "op@felhom.eu")); got != 2 {
t.Fatalf("operator mails = %d, want 2", got)
}
*sent = nil
d.ProcessEvent("c1", "app_unexplained_upload_cleared", "info", "qbittorrent: stopped", `{"app":"qbittorrent","cleared":true,"app_mbps":-1}`, "controller")
if len(*sent) != 0 {
t.Fatalf("a clear (info) mailed %d time(s)", len(*sent))
}
_, body := FormatOperatorEmail("c1", "app_unexplained_upload", "warning", "msg", up)
want := "Watch: qbittorrent — upload 10136.8 Mbit/s for 15 min; the app counted 0.0 Mbit/s (asked: qbittorrent, viewers 0); the box's line: at least 47 Mbit/s"
if !strings.Contains(body, want) {
t.Fatalf("body lacks %q:\n%s", want, body)
}
_, cpu := FormatOperatorEmail("c1", "app_unexplained_cpu", "warning", "msg", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"explainer":"plex transcodes"}`)
if !strings.Contains(cpu, "Watch: plex — 380% CPU (of one core) for 30 min; asked: plex transcodes, transcodes 0") {
t.Fatalf("cpu body:\n%s", cpu)
}
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
if !IsOperatorOnly(et) {
t.Errorf("%s is not operator-only", et)
}
}
}