v0.342.0 (unreleased): the upload and CPU watch — an app that sends or burns far more than its own numbers explain is one operator-only alarm; the line learnt from the off-site push; qBittorrent's default upload limit (never over the household's)
gates / gates (push) Successful in 1m17s
gates / gates (push) Successful in 1m17s
internal/upwatch: exact per-container counters from the Docker Engine API, per-app judging (upload >= max(5 Mbit/s, 25% of the line) for 15 min; CPU >= max(150%, 50% x cores) for 30 min), explainers qBittorrent / Transmission / Plex, app_unexplained_upload / app_unexplained_cpu + _cleared (hub 0.169.0). The off-site push is sampled once a second for the line (a lower bound). qBittorrent limit = 50% of the line, the household's limit always wins. 20 red-proofs. Spike + design: felhom.eu audits/SPIKE-upload-watch-2026-10-23.md, architecture/08 section 9. Not released.
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
// Package upwatch is the upload and CPU watch (felhom.eu `architecture/08-alarm-ladder.md` §9 [DESIGN — proposed],
|
||||
// spike `audits/SPIKE-upload-watch-2026-10-23.md`). Once a minute it reads every app container's EXACT sent bytes and
|
||||
// CPU time, adds them up per app, and when an app sends a lot (or burns CPU) for a long time it ASKS THE APP whether
|
||||
// that is its normal work — qBittorrent's and Transmission's own upload counters, Plex's playing and transcoding
|
||||
// sessions. Work the app explains is normal use. Work it does not explain is the sign of a hacked app (DooPlex's Plex
|
||||
// sent 474 GB in one day with no viewer, 2026-10-23) and becomes ONE operator-only event, cleared when it stops.
|
||||
//
|
||||
// It also learns the box's upload line from what the box already does (its own off-site push, line.go) and gives
|
||||
// qBittorrent a default upload limit from it, never over a limit the household set (qblimit.go).
|
||||
//
|
||||
// SECRETS: app passwords and tokens are read at use (the mediawire seams, `09` §3 decision 248), sent only to the
|
||||
// app's own container, never stored, never logged.
|
||||
package upwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sample is one container's cumulative counters at one moment.
|
||||
type Sample struct {
|
||||
Container string
|
||||
Stack string // the compose project; "" for a container outside any stack
|
||||
TxBytes uint64 // bytes sent on every network of the container, since it started
|
||||
CPUNanos uint64 // CPU time used, since it started
|
||||
Started string // the container's start time: a change means the counters restarted from zero
|
||||
}
|
||||
|
||||
// Source returns one Sample per running container.
|
||||
type Source interface {
|
||||
Samples(ctx context.Context) ([]Sample, error)
|
||||
}
|
||||
|
||||
// DockerAPI reads the counters from the Docker Engine API on the guest's socket.
|
||||
//
|
||||
// WHY NOT `docker stats` (what internal/metrics uses): its NetIO is text with three significant digits ("1.23GB"), so
|
||||
// at a large total one step is ~10 MB — 1.3 Mbit/s at a one-minute sample. The Engine API gives integers; measured on
|
||||
// 9202 it equals the kernel's own counter in the container's network namespace to the byte
|
||||
// (`audits/SPIKE-upload-watch-2026-10-23/s4-qb-seed-nolimit.jsonl`: proc_tx == engine_tx).
|
||||
type DockerAPI struct {
|
||||
Socket string // default /var/run/docker.sock
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (d *DockerAPI) http() *http.Client {
|
||||
if d.client == nil {
|
||||
sock := d.Socket
|
||||
if sock == "" {
|
||||
sock = "/var/run/docker.sock"
|
||||
}
|
||||
d.client = &http.Client{
|
||||
Timeout: 20 * time.Second,
|
||||
Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
var dl net.Dialer
|
||||
return dl.DialContext(ctx, "unix", sock) // dialled by path every time: sockheal may re-create the file
|
||||
}},
|
||||
}
|
||||
}
|
||||
return d.client
|
||||
}
|
||||
|
||||
func (d *DockerAPI) get(ctx context.Context, path string, out interface{}) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "http://docker"+path, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := d.http().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("docker API %s: HTTP %d %s", path, resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(out)
|
||||
}
|
||||
|
||||
// Samples lists the running containers and reads each one's counters (one-shot stats: no one-second wait).
|
||||
func (d *DockerAPI) Samples(ctx context.Context) ([]Sample, error) {
|
||||
var list []struct {
|
||||
ID string `json:"Id"`
|
||||
Names []string `json:"Names"`
|
||||
Labels map[string]string `json:"Labels"`
|
||||
}
|
||||
if err := d.get(ctx, "/containers/json", &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Sample, 0, len(list))
|
||||
for _, c := range list {
|
||||
var st struct {
|
||||
Read string `json:"read"`
|
||||
Networks map[string]struct {
|
||||
TxBytes uint64 `json:"tx_bytes"`
|
||||
} `json:"networks"`
|
||||
CPUStats struct {
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
} `json:"cpu_stats"`
|
||||
}
|
||||
if err := d.get(ctx, "/containers/"+c.ID+"/stats?stream=false&one-shot=true", &st); err != nil {
|
||||
continue // the container stopped between the list and the read: it is simply not sampled this minute
|
||||
}
|
||||
var started struct {
|
||||
State struct {
|
||||
StartedAt string `json:"StartedAt"`
|
||||
} `json:"State"`
|
||||
}
|
||||
_ = d.get(ctx, "/containers/"+c.ID+"/json", &started)
|
||||
s := Sample{Stack: c.Labels["com.docker.compose.project"], CPUNanos: st.CPUStats.CPUUsage.TotalUsage, Started: started.State.StartedAt}
|
||||
if len(c.Names) > 0 {
|
||||
s.Container = strings.TrimPrefix(c.Names[0], "/")
|
||||
}
|
||||
for _, n := range st.Networks {
|
||||
s.TxBytes += n.TxBytes
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user