v0.342.0 (unreleased): the upload and CPU watch — an app that sends or burns far more than its own numbers explain is one operator-only alarm; the line learnt from the off-site push; qBittorrent's default upload limit (never over the household's)
gates / gates (push) Successful in 1m17s
gates / gates (push) Successful in 1m17s
internal/upwatch: exact per-container counters from the Docker Engine API, per-app judging (upload >= max(5 Mbit/s, 25% of the line) for 15 min; CPU >= max(150%, 50% x cores) for 30 min), explainers qBittorrent / Transmission / Plex, app_unexplained_upload / app_unexplained_cpu + _cleared (hub 0.169.0). The off-site push is sampled once a second for the line (a lower bound). qBittorrent limit = 50% of the line, the household's limit always wins. 20 red-proofs. Spike + design: felhom.eu audits/SPIKE-upload-watch-2026-10-23.md, architecture/08 section 9. Not released.
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
package upwatch
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── The upload line, learnt from what the box already does ────────────────────────────────────────────────────────
|
||||
//
|
||||
// The box's off-site push (restic, inside this controller's own container) is the only big upload the box makes on
|
||||
// its own. While it runs, the controller samples its OWN network counter once a second and keeps the best 10-second
|
||||
// rate. Over nights, the highest such rate is a LOWER BOUND of the household's upload line — the push may not fill
|
||||
// the line (the family uploads too, the Storage Box may be slow), so the page says „at least".
|
||||
//
|
||||
// Measured (spike §2): at a one-minute sample the push looked like 0.3–37 Mbit/s, because most nights it moves a few
|
||||
// megabytes in under a minute — the one-minute counter shows the data size, not the line. Hence a 1-second sampler,
|
||||
// and a night only COUNTS when its best 10 seconds moved ≥ MinWindowBytes: a short push says nothing about the line.
|
||||
|
||||
// MinWindowBytes is the least a push must move in its best 10 seconds to say anything about the line (≥ 8 Mbit/s).
|
||||
const MinWindowBytes = 10 << 20
|
||||
|
||||
// KeepNights is how many nights' records the line is the maximum of.
|
||||
const KeepNights = 14
|
||||
|
||||
// LineRecord is one push's measurement.
|
||||
type LineRecord struct {
|
||||
At string `json:"at"` // RFC3339, the push's end
|
||||
Peak10Mbps float64 `json:"peak10_mbps"` // the best 10-second rate
|
||||
Bytes uint64 `json:"bytes"` // everything the container sent during the push
|
||||
Seconds int `json:"seconds"`
|
||||
Counts bool `json:"counts"` // the best 10 seconds moved ≥ MinWindowBytes
|
||||
}
|
||||
|
||||
// LineFromRecords is the line estimate: the highest counting peak over the kept nights. ok=false while none counts.
|
||||
func LineFromRecords(recs []LineRecord) (float64, bool) {
|
||||
best, ok := 0.0, false
|
||||
for _, r := range recs {
|
||||
if r.Counts && r.Peak10Mbps > best {
|
||||
best, ok = r.Peak10Mbps, true
|
||||
}
|
||||
}
|
||||
return best, ok
|
||||
}
|
||||
|
||||
// AddRecord appends one push and keeps the newest KeepNights.
|
||||
func AddRecord(recs []LineRecord, r LineRecord) []LineRecord {
|
||||
recs = append(recs, r)
|
||||
if len(recs) > KeepNights {
|
||||
recs = recs[len(recs)-KeepNights:]
|
||||
}
|
||||
return recs
|
||||
}
|
||||
|
||||
// OwnTxBytes sums the bytes sent on every interface but loopback in THIS process's network namespace
|
||||
// (/proc/self/net/dev; the 9th field of an interface line is transmit bytes).
|
||||
func OwnTxBytes() (uint64, error) {
|
||||
return txFromNetDev("/proc/self/net/dev")
|
||||
}
|
||||
|
||||
func txFromNetDev(path string) (uint64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
var sum uint64
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
i := strings.Index(line, ":")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(line[:i]) == "lo" {
|
||||
continue
|
||||
}
|
||||
fs := strings.Fields(line[i+1:])
|
||||
if len(fs) < 9 {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseUint(fs[8], 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s: %q: %w", path, line, err)
|
||||
}
|
||||
sum += v
|
||||
}
|
||||
return sum, sc.Err()
|
||||
}
|
||||
|
||||
// Point is one reading of the counter.
|
||||
type Point struct {
|
||||
At time.Time
|
||||
V uint64
|
||||
}
|
||||
|
||||
// Summarize turns a push's readings into its record: total bytes, seconds, and the best rate over any window of at
|
||||
// least 10 seconds (for each start, the first reading ≥ 10 s later). A counter that went down is no measurement.
|
||||
func Summarize(pts []Point, end time.Time) LineRecord {
|
||||
r := LineRecord{At: end.UTC().Format(time.RFC3339)}
|
||||
if len(pts) < 2 {
|
||||
return r
|
||||
}
|
||||
first, last := pts[0], pts[len(pts)-1]
|
||||
if last.V >= first.V {
|
||||
r.Bytes = last.V - first.V
|
||||
}
|
||||
r.Seconds = int(last.At.Sub(first.At).Seconds() + 0.5)
|
||||
var bestBytes uint64
|
||||
var bestRate float64
|
||||
j := 0
|
||||
for i := range pts {
|
||||
if j < i {
|
||||
j = i
|
||||
}
|
||||
for j < len(pts) && pts[j].At.Sub(pts[i].At) < 10*time.Second {
|
||||
j++
|
||||
}
|
||||
if j >= len(pts) {
|
||||
break
|
||||
}
|
||||
if pts[j].V < pts[i].V {
|
||||
continue
|
||||
}
|
||||
b := pts[j].V - pts[i].V
|
||||
rate := float64(b) * 8 / pts[j].At.Sub(pts[i].At).Seconds() / 1e6
|
||||
if rate > bestRate {
|
||||
bestRate, bestBytes = rate, b
|
||||
}
|
||||
}
|
||||
r.Peak10Mbps = round2(bestRate)
|
||||
r.Counts = bestBytes >= MinWindowBytes
|
||||
return r
|
||||
}
|
||||
|
||||
// PushSampler samples a byte counter once a second between Start and the stop function it returns.
|
||||
type PushSampler struct {
|
||||
Read func() (uint64, error) // default OwnTxBytes
|
||||
Every time.Duration // default 1 s
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Start begins sampling; the returned stop ends it and returns the push's record.
|
||||
func (p *PushSampler) Start() (stop func() LineRecord) {
|
||||
read := p.Read
|
||||
if read == nil {
|
||||
read = OwnTxBytes
|
||||
}
|
||||
every := p.Every
|
||||
if every <= 0 {
|
||||
every = time.Second
|
||||
}
|
||||
now := p.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var pts []Point
|
||||
take := func() {
|
||||
if v, err := read(); err == nil {
|
||||
mu.Lock()
|
||||
pts = append(pts, Point{now(), v})
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
take()
|
||||
done := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finished)
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-t.C:
|
||||
take()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() LineRecord {
|
||||
close(done)
|
||||
<-finished
|
||||
take()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return Summarize(pts, now())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user