v0.342.0 (unreleased): the upload and CPU watch — an app that sends or burns far more than its own numbers explain is one operator-only alarm; the line learnt from the off-site push; qBittorrent's default upload limit (never over the household's)

internal/upwatch: exact per-container counters from the Docker Engine API, per-app judging (upload >= max(5 Mbit/s, 25% of the line) for 15 min; CPU >= max(150%, 50% x cores) for 30 min), explainers qBittorrent / Transmission / Plex, app_unexplained_upload / app_unexplained_cpu + _cleared (hub 0.169.0). The off-site push is sampled once a second for the line (a lower bound). qBittorrent limit = 50% of the line, the household's limit always wins. 20 red-proofs. Spike + design: felhom.eu audits/SPIKE-upload-watch-2026-10-23.md, architecture/08 section 9. Not released.
2026-10-23 21:26:09 +02:00
parent 277e99839d
commit a9c1997fbc
35 changed files with 2438 additions and 0 deletions
@@ -685,6 +685,15 @@ func main() {
 	// R-550: an interrupted restore is raised once, now that the notifier exists.
 	reportInterruptedRestore(notifier, interruptedRestore)

+	// The upload and CPU watch (internal/upwatch; felhom.eu `08` §9, operator ruling 2026-10-23). Built here, where the
+	// notifier first exists, and its push measurement handed to the off-site backup BEFORE any scheduler can start a
+	// push. Wiring pinned by TestUpwatchIsWiredAtStartup.
+	upwatchSvc := newUpwatch(sett, stackMgr, backupMgr, metricsStore, notifier, logger)
+	if backupMgr != nil {
+		backupMgr.SetOffboxPushObserver(upwatchSvc.PushObserver)
+	}
+	startUpwatch(ctx, upwatchSvc, logger)
+
 	// R-97a: wire the quiesce loop's hub-event seam. It MUST happen here and not in
 	// startQuiesceLoop, because the notifier is constructed after it — and it must happen at all,
 	// or the seam is inert (the "built but never wired" trap, four recorded instances). Reachability
@@ -2044,6 +2053,8 @@ func main() {
 	// URL is derived from it, so two builds of one version can never share a cache entry. Init-only,
 	// before the HTTP server starts.
 	webServer.SetBuildIdentity(GitCommit, BuildTime)
+	// The upload watch's two page lines: the measured line (settings → system) and qBittorrent's limit (its app page).
+	webServer.SetUpwatchView(upwatchSvc.View)
 	// R-871 (decision 110): the missed-backup banner reads the night ledger and the box's own on/off record.
 	webServer.SetMissedBackupBanner(webNightLedger, metricsStore)
 	// Migration done-hook: a decommission-initiated migration finalizes the source decommission on
@@ -0,0 +1,105 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/mediawire"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/upwatch"
+)
+
+// qbLimitShare is qBittorrent's default share of the measured upload line — CC's pick, open for the operator (decide
+// row in felhom.eu STATUS): 50 %.
+const qbLimitShare = 0.5
+
+// baselineMinMinutes: an app the box cannot ask is judged only once its history holds ~20 hours of minutes.
+const baselineMinMinutes = 1200
+
+// newUpwatch builds the upload and CPU watch (internal/upwatch; felhom.eu `08` §9). It does not start it.
+func newUpwatch(sett *settings.Settings, stackMgr *stacks.Manager, backupMgr *backup.Manager, metricsStore *metrics.MetricsStore,
+	notifier *notify.Notifier, logger *log.Logger) *upwatch.Service {
+	box := mediawire.DockerBox{}
+	qb := &upwatch.QBittorrent{Box: box}
+	svc := &upwatch.Service{Store: sett, QB: qb, QBRunning: func() bool { return box.Running("qbittorrent") }, QBShare: qbLimitShare, Log: logger}
+	w := &upwatch.Watcher{
+		Source: &upwatch.DockerAPI{},
+		Explainers: map[string]upwatch.Explainer{
+			"qbittorrent":  qb,
+			"transmission": &upwatch.Transmission{Box: box},
+			"plex":         &upwatch.Plex{Token: func() string { return mediawire.PlexToken(box) }, URL: mediawire.PlexURL},
+		},
+		Rules: upwatch.DefaultRules(),
+		Line:  svc.Line,
+		Busy: func(stack string) bool {
+			if backupMgr != nil && backupMgr.IsRunning() {
+				return true
+			}
+			if st, ok := stackMgr.GetStack(stack); ok && st != nil && (st.Updating || st.Deploying) {
+				return true
+			}
+			return false
+		},
+		Emit: notifier.NotifyUnexplained,
+		Log:  logger,
+	}
+	if metricsStore != nil {
+		w.Baseline = dailyBaseline(metricsStore)
+	}
+	svc.Watcher = w
+	return svc
+}
+
+// dailyBaseline caches each app's p95 for a day (a 7-day query is a few hundred thousand rows).
+func dailyBaseline(ms *metrics.MetricsStore) func(string, []string) (float64, bool) {
+	type entry struct {
+		at  time.Time
+		p95 float64
+		ok  bool
+	}
+	var mu sync.Mutex
+	cache := map[string]entry{}
+	return func(stack string, containers []string) (float64, bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if e, ok := cache[stack]; ok && time.Since(e.at) < 24*time.Hour {
+			return e.p95, e.ok
+		}
+		p95, minutes, err := ms.StackTxP95Mbps(containers, time.Now().Add(-7*24*time.Hour))
+		e := entry{at: time.Now(), p95: p95, ok: err == nil && minutes >= baselineMinMinutes}
+		cache[stack] = e
+		return e.p95, e.ok
+	}
+}
+
+// startUpwatch runs the minute watcher and the 15-minute qBittorrent limit rule until ctx ends.
+func startUpwatch(ctx context.Context, svc *upwatch.Service, logger *log.Logger) {
+	go svc.Watcher.Run(ctx, time.Minute)
+	go func() {
+		t := time.NewTicker(15 * time.Minute)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				svc.QBLimitNow(ctx)
+			}
+		}
+	}()
+	l, ok := svc.Line()
+	logger.Printf("[INFO] [upwatch] upload and CPU watch started (line %s, qBittorrent share %.0f %%)", lineText(l, ok), qbLimitShare*100)
+}
+
+func lineText(l float64, ok bool) string {
+	if !ok {
+		return "unknown — learning from the off-site push"
+	}
+	return "at least " + upwatch.MbpsText(int64(l*1e6/8)) + " Mbit/s"
+}
@@ -0,0 +1,52 @@
+package main
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/upwatch"
+)
+
+// The upload watch is built, started, handed to the off-site push BEFORE the scheduler can start one, and its view
+// handed to the pages — a seam built and never wired is this project's commonest defect.
+// RED-PROOF: delete the SetOffboxPushObserver line → fails; move it below the scheduler's start → fails.
+func TestUpwatchIsWiredAtStartup(t *testing.T) {
+	lines, _, _ := slice4CallLines(t)
+	for _, fn := range []string{"newUpwatch", "SetOffboxPushObserver", "startUpwatch", "SetUpwatchView"} {
+		if len(lines[fn]) == 0 {
+			t.Fatalf("main.go never calls %s", fn)
+		}
+	}
+	if lines["SetOffboxPushObserver"][0] > lines["Daily"][0] {
+		t.Fatalf("SetOffboxPushObserver (line %d) comes after the first scheduled job (line %d)", lines["SetOffboxPushObserver"][0], lines["Daily"][0])
+	}
+}
+
+// newUpwatch asks the three apps the spike proved, reports through the notifier, and reads exact counters.
+// RED-PROOF: drop the transmission entry → fails.
+func TestNewUpwatch_ExplainersAndEmit(t *testing.T) {
+	lg := log.New(io.Discard, "", 0)
+	sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), lg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc := newUpwatch(sett, nil, nil, nil, notify.New("", "", "", sett, lg, false), lg)
+	for _, a := range []string{"qbittorrent", "transmission", "plex"} {
+		if _, ok := svc.Watcher.Explainers[a]; !ok {
+			t.Errorf("no explainer for %s", a)
+		}
+	}
+	if svc.Watcher.Emit == nil || svc.Watcher.Line == nil || svc.QB == nil || svc.QBShare != 0.5 {
+		t.Fatalf("service not wired: %+v", svc)
+	}
+	if _, ok := svc.Watcher.Source.(*upwatch.DockerAPI); !ok {
+		t.Fatal("the watch must read the Engine API's exact counters, not docker stats' text")
+	}
+	if svc.Watcher.Baseline != nil {
+		t.Fatal("no metrics store, yet a baseline: an app with no explainer would be judged without history")
+	}
+}