hub 0.169.0 (unreleased, not built): the upload watch's four events — allowed, operator-only, per-app cooldown, one readable Watch line in the operator mail
gates / gates (push) Successful in 12m47s
gates / gates (push) Successful in 12m47s
app_unexplained_upload / app_unexplained_cpu (warning) and their _cleared (info), from controller v0.342.0 (08 section 9, operator ruling 2026-10-23). Red-proved: operator-only, the render line, the allow-list.
This commit is contained in:
@@ -1,3 +1,16 @@
|
||||
## v0.169.0 (unreleased — committed 2026-10-23 night, NOT built, NOT deployed) — the upload watch's four events
|
||||
|
||||
- **`app_unexplained_upload`, `app_unexplained_cpu` (warning) and their `_cleared` (info)** from controller v0.342.0's
|
||||
upload and CPU watch (`08` §9 [DESIGN — proposed], operator ruling 2026-10-23 19:22): an app sent or burnt far more
|
||||
than its own numbers explain — the sign of a hacked app (Plex2Shell). In `allowedEventTypes` AND `operatorOnlyEvents`
|
||||
(the R-388 rule: the household can do nothing about it), per-app cooldown (`perAppCooldownEvents` — two hacked apps
|
||||
are two alarms; `TestR389_TheAllowListHasExactlyOneMember` widened with its dated reason). The operator mail carries
|
||||
one `Watch:` line (`renderUnexplained`): the rate, what the app counted (or „no app number to compare"), the minutes,
|
||||
what was asked, the box's measured line. A clear is info: recorded, not mailed. Tests
|
||||
`TestUpwatchEvents_OperatorOnlyWithAReadableLine`, `TestUpwatchEventTypesAreAllowedAndOperatorOnly` — red-proved
|
||||
(dropped from operatorOnlyEvents → the household was mailed; the render call removed → no Watch line; dropped from
|
||||
the allow-list → fails).
|
||||
|
||||
## v0.168.0 (unreleased — built 2026-10-23 night, NOT deployed; the deploy is the operator's yes) — the last plain tokens sealed (R-975, R-1241)
|
||||
|
||||
- **R-975 / R-1241's Felhom half.** The Cloudflare tunnel token, the Cloudflare API token and the git token lived in
|
||||
|
||||
@@ -2410,6 +2410,13 @@ var allowedEventTypes = map[string]bool{
|
||||
// definition (a correction, an update, a restore, a settings change) — nothing was written, the app runs on its last
|
||||
// good compose. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||||
"app_definition_refused": true,
|
||||
// The upload and CPU watch (hub v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23): an app sent or burnt
|
||||
// far more than its own numbers explain — the sign of a hacked app (Plex2Shell) — and the clear when it stopped.
|
||||
// Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||||
"app_unexplained_upload": true,
|
||||
"app_unexplained_upload_cleared": true,
|
||||
"app_unexplained_cpu": true,
|
||||
"app_unexplained_cpu_cleared": true,
|
||||
// Special
|
||||
"test": true,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
|
||||
)
|
||||
|
||||
// The four upload-watch types are accepted from a box (an unlisted type is a 400 and is dropped) AND operator-only —
|
||||
// both registers, same commit. RED-PROOF: drop one from allowedEventTypes → fails.
|
||||
func TestUpwatchEventTypesAreAllowedAndOperatorOnly(t *testing.T) {
|
||||
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
|
||||
if !allowedEventTypes[et] {
|
||||
t.Errorf("%s is not in allowedEventTypes — POST /event would 400 and the alarm would be lost", et)
|
||||
}
|
||||
if !notify.IsOperatorOnly(et) {
|
||||
t.Errorf("%s is not operator-only", et)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -490,6 +490,9 @@ var perAppCooldownEvents = map[string]bool{
|
||||
// v0.150.0 (R-1007): one app's repair after an unclean boot, no digest behind it — two apps on one boot are two
|
||||
// records, and a lost photo on the second must not be swallowed by the first's cooldown.
|
||||
"app_repaired_after_crash": true,
|
||||
// v0.169.0 (the upload watch): one app's unexplained upload or CPU, no digest — two hacked apps are two alarms.
|
||||
"app_unexplained_upload": true,
|
||||
"app_unexplained_cpu": true,
|
||||
}
|
||||
|
||||
// perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The
|
||||
@@ -892,6 +895,13 @@ var operatorOnlyEvents = map[string]bool{
|
||||
// of empty files found / repaired / still broken / originals lost. The household learns of a lost photo on its own
|
||||
// app page (name and date, on its box); THESE counts are the operator's. Registered in the same commit that mints it.
|
||||
"app_repaired_after_crash": true,
|
||||
// The upload and CPU watch (v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23 — the R-388 rule): an
|
||||
// app sending or burning far more than its own numbers explain. The household can do nothing about a hacked app but
|
||||
// wait for support; the operator decides. The clears too (belt, as for the whole-guest pair). Same commit.
|
||||
"app_unexplained_upload": true,
|
||||
"app_unexplained_upload_cleared": true,
|
||||
"app_unexplained_cpu": true,
|
||||
"app_unexplained_cpu_cleared": true,
|
||||
// R-971 (v0.157.0, controller v0.328.0, operator decision D2 = A). The box's catalog allow-list refused an app's
|
||||
// definition: someone pushed a catalog change asking for container rights the box does not give. The household can
|
||||
// do nothing; its app page says the app keeps its last good definition. Registered in the same commit that mints it.
|
||||
|
||||
@@ -72,7 +72,8 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
|
||||
// v0.122.0 once more (R-659): a stranded held app is one app's event with no digest.
|
||||
// v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest.
|
||||
// v0.150.0 once more (R-1007): an app's repair after an unclean boot is one app's event with no digest.
|
||||
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"}
|
||||
// v0.169.0 once more (the upload watch): an app's unexplained upload or CPU is one app's event with no digest.
|
||||
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_unexplained_cpu", "app_unexplained_upload", "app_update_held", "app_update_undone"}
|
||||
ok := len(perAppCooldownEvents) == len(want)
|
||||
for _, w := range want {
|
||||
ok = ok && perAppCooldownEvents[w]
|
||||
@@ -82,7 +83,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
|
||||
for k := range perAppCooldownEvents {
|
||||
got = append(got, k)
|
||||
}
|
||||
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+
|
||||
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_unexplained_cpu app_unexplained_upload app_update_held app_update_undone]. Adding a member is the "+
|
||||
"fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+
|
||||
"disk sends one digest, not one mail per app. Read the fence before widening this.", got)
|
||||
}
|
||||
|
||||
@@ -70,6 +70,13 @@ Message: %s`, customerID, eventType, severity, now, message)
|
||||
}
|
||||
}
|
||||
|
||||
// The upload and CPU watch (v0.169.0): what the app sent or burnt, what it said it did, and for how long.
|
||||
if eventType == "app_unexplained_upload" || eventType == "app_unexplained_cpu" {
|
||||
if line, ok := renderUnexplained(detailsJSON); ok {
|
||||
body += "\n" + line
|
||||
}
|
||||
}
|
||||
|
||||
if detailsJSON != "" && detailsJSON != "{}" {
|
||||
body += fmt.Sprintf("\nDetails: %s", detailsJSON)
|
||||
}
|
||||
@@ -432,6 +439,37 @@ func stackNameOf(detailsJSON string) string {
|
||||
return d.StackName
|
||||
}
|
||||
|
||||
// renderUnexplained is the upload watch's operator line (v0.169.0, controller upwatch.Details).
|
||||
func renderUnexplained(detailsJSON string) (string, bool) {
|
||||
var d struct {
|
||||
App string `json:"app"`
|
||||
Kind string `json:"kind"`
|
||||
Minutes int `json:"minutes"`
|
||||
RateMbps float64 `json:"rate_mbps"`
|
||||
AppMbps float64 `json:"app_mbps"`
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
Explainer string `json:"explainer"`
|
||||
Viewers int `json:"viewers"`
|
||||
Transcodes int `json:"transcodes"`
|
||||
LineMbps float64 `json:"line_mbps"`
|
||||
}
|
||||
if json.Unmarshal([]byte(detailsJSON), &d) != nil || d.App == "" {
|
||||
return "", false
|
||||
}
|
||||
if d.Kind == "cpu" {
|
||||
return fmt.Sprintf("Watch: %s — %.0f%% CPU (of one core) for %d min; asked: %s, transcodes %d", d.App, d.CPUPercent, d.Minutes, d.Explainer, d.Transcodes), true
|
||||
}
|
||||
said := "no app number to compare"
|
||||
if d.AppMbps >= 0 {
|
||||
said = fmt.Sprintf("the app counted %.1f Mbit/s", d.AppMbps)
|
||||
}
|
||||
line := fmt.Sprintf("Watch: %s — upload %.1f Mbit/s for %d min; %s (asked: %s, viewers %d)", d.App, d.RateMbps, d.Minutes, said, d.Explainer, d.Viewers)
|
||||
if d.LineMbps > 0 {
|
||||
line += fmt.Sprintf("; the box's line: at least %.0f Mbit/s", d.LineMbps)
|
||||
}
|
||||
return line, true
|
||||
}
|
||||
|
||||
// renderAppRepair is app_repaired_after_crash's operator line (R-1007): the counts, the boot and the duration.
|
||||
func renderAppRepair(detailsJSON string) (string, bool) {
|
||||
var d struct {
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The upload and CPU watch's events (hub v0.169.0, controller v0.342.0) reach the OPERATOR and never the household — even
|
||||
// one that enabled them by name; the alarm mails once with a readable line; the clear (info) is recorded, not mailed.
|
||||
// RED-PROOF: drop app_unexplained_upload from operatorOnlyEvents → the household gets a mail → fails; drop the
|
||||
// renderUnexplained call → the body lacks the Watch line → fails.
|
||||
func TestUpwatchEvents_OperatorOnlyWithAReadableLine(t *testing.T) {
|
||||
st := newDispStore(t)
|
||||
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"app_unexplained_upload", "app_unexplained_cpu", "node_down"}, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
|
||||
sent := captureSeam(d)
|
||||
up := `{"app":"qbittorrent","stack_name":"qbittorrent","kind":"upload","minutes":15,"rate_mbps":10136.77,"app_mbps":0,"explainer":"qbittorrent","viewers":0,"transcodes":0,"line_mbps":47}`
|
||||
d.ProcessEvent("c1", "app_unexplained_upload", "warning", "qbittorrent: ...", up, "controller")
|
||||
d.ProcessEvent("c1", "app_unexplained_cpu", "warning", "plex: ...", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"app_mbps":-1,"explainer":"plex transcodes","transcodes":0}`, "controller")
|
||||
if got := mailsFor(*sent, "cust@example.com"); len(got) != 0 {
|
||||
t.Fatalf("the household got %d mail(s) — a hacked app is the operator's to handle", len(got))
|
||||
}
|
||||
if got := len(mailsFor(*sent, "op@felhom.eu")); got != 2 {
|
||||
t.Fatalf("operator mails = %d, want 2", got)
|
||||
}
|
||||
*sent = nil
|
||||
d.ProcessEvent("c1", "app_unexplained_upload_cleared", "info", "qbittorrent: stopped", `{"app":"qbittorrent","cleared":true,"app_mbps":-1}`, "controller")
|
||||
if len(*sent) != 0 {
|
||||
t.Fatalf("a clear (info) mailed %d time(s)", len(*sent))
|
||||
}
|
||||
_, body := FormatOperatorEmail("c1", "app_unexplained_upload", "warning", "msg", up)
|
||||
want := "Watch: qbittorrent — upload 10136.8 Mbit/s for 15 min; the app counted 0.0 Mbit/s (asked: qbittorrent, viewers 0); the box's line: at least 47 Mbit/s"
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("body lacks %q:\n%s", want, body)
|
||||
}
|
||||
_, cpu := FormatOperatorEmail("c1", "app_unexplained_cpu", "warning", "msg", `{"app":"plex","kind":"cpu","minutes":30,"cpu_percent":380,"explainer":"plex transcodes"}`)
|
||||
if !strings.Contains(cpu, "Watch: plex — 380% CPU (of one core) for 30 min; asked: plex transcodes, transcodes 0") {
|
||||
t.Fatalf("cpu body:\n%s", cpu)
|
||||
}
|
||||
for _, et := range []string{"app_unexplained_upload", "app_unexplained_upload_cleared", "app_unexplained_cpu", "app_unexplained_cpu_cleared"} {
|
||||
if !IsOperatorOnly(et) {
|
||||
t.Errorf("%s is not operator-only", et)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user