v0.342.0 (unreleased): the upload and CPU watch — an app that sends or burns far more than its own numbers explain is one operator-only alarm; the line learnt from the off-site push; qBittorrent's default upload limit (never over the household's)

internal/upwatch: exact per-container counters from the Docker Engine API, per-app judging (upload >= max(5 Mbit/s, 25% of the line) for 15 min; CPU >= max(150%, 50% x cores) for 30 min), explainers qBittorrent / Transmission / Plex, app_unexplained_upload / app_unexplained_cpu + _cleared (hub 0.169.0). The off-site push is sampled once a second for the line (a lower bound). qBittorrent limit = 50% of the line, the household's limit always wins. 20 red-proofs. Spike + design: felhom.eu audits/SPIKE-upload-watch-2026-10-23.md, architecture/08 section 9. Not released.
2026-10-23 21:26:09 +02:00
parent 277e99839d
commit a9c1997fbc
35 changed files with 2438 additions and 0 deletions
@@ -0,0 +1,105 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/mediawire"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/upwatch"
+)
+
+// qbLimitShare is qBittorrent's default share of the measured upload line — CC's pick, open for the operator (decide
+// row in felhom.eu STATUS): 50 %.
+const qbLimitShare = 0.5
+
+// baselineMinMinutes: an app the box cannot ask is judged only once its history holds ~20 hours of minutes.
+const baselineMinMinutes = 1200
+
+// newUpwatch builds the upload and CPU watch (internal/upwatch; felhom.eu `08` §9). It does not start it.
+func newUpwatch(sett *settings.Settings, stackMgr *stacks.Manager, backupMgr *backup.Manager, metricsStore *metrics.MetricsStore,
+	notifier *notify.Notifier, logger *log.Logger) *upwatch.Service {
+	box := mediawire.DockerBox{}
+	qb := &upwatch.QBittorrent{Box: box}
+	svc := &upwatch.Service{Store: sett, QB: qb, QBRunning: func() bool { return box.Running("qbittorrent") }, QBShare: qbLimitShare, Log: logger}
+	w := &upwatch.Watcher{
+		Source: &upwatch.DockerAPI{},
+		Explainers: map[string]upwatch.Explainer{
+			"qbittorrent":  qb,
+			"transmission": &upwatch.Transmission{Box: box},
+			"plex":         &upwatch.Plex{Token: func() string { return mediawire.PlexToken(box) }, URL: mediawire.PlexURL},
+		},
+		Rules: upwatch.DefaultRules(),
+		Line:  svc.Line,
+		Busy: func(stack string) bool {
+			if backupMgr != nil && backupMgr.IsRunning() {
+				return true
+			}
+			if st, ok := stackMgr.GetStack(stack); ok && st != nil && (st.Updating || st.Deploying) {
+				return true
+			}
+			return false
+		},
+		Emit: notifier.NotifyUnexplained,
+		Log:  logger,
+	}
+	if metricsStore != nil {
+		w.Baseline = dailyBaseline(metricsStore)
+	}
+	svc.Watcher = w
+	return svc
+}
+
+// dailyBaseline caches each app's p95 for a day (a 7-day query is a few hundred thousand rows).
+func dailyBaseline(ms *metrics.MetricsStore) func(string, []string) (float64, bool) {
+	type entry struct {
+		at  time.Time
+		p95 float64
+		ok  bool
+	}
+	var mu sync.Mutex
+	cache := map[string]entry{}
+	return func(stack string, containers []string) (float64, bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if e, ok := cache[stack]; ok && time.Since(e.at) < 24*time.Hour {
+			return e.p95, e.ok
+		}
+		p95, minutes, err := ms.StackTxP95Mbps(containers, time.Now().Add(-7*24*time.Hour))
+		e := entry{at: time.Now(), p95: p95, ok: err == nil && minutes >= baselineMinMinutes}
+		cache[stack] = e
+		return e.p95, e.ok
+	}
+}
+
+// startUpwatch runs the minute watcher and the 15-minute qBittorrent limit rule until ctx ends.
+func startUpwatch(ctx context.Context, svc *upwatch.Service, logger *log.Logger) {
+	go svc.Watcher.Run(ctx, time.Minute)
+	go func() {
+		t := time.NewTicker(15 * time.Minute)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				svc.QBLimitNow(ctx)
+			}
+		}
+	}()
+	l, ok := svc.Line()
+	logger.Printf("[INFO] [upwatch] upload and CPU watch started (line %s, qBittorrent share %.0f %%)", lineText(l, ok), qbLimitShare*100)
+}
+
+func lineText(l float64, ok bool) string {
+	if !ok {
+		return "unknown — learning from the off-site push"
+	}
+	return "at least " + upwatch.MbpsText(int64(l*1e6/8)) + " Mbit/s"
+}