440 lines
14 KiB
Go
440 lines
14 KiB
Go
// Package linetest is the weekly upload line-speed test (R-1247; operator ruling 2026-10-24 07:26).
|
||
//
|
||
// Why: the controller's upload watch and qBittorrent's default limit (50 % of the line, D41) need the household's
|
||
// upload line. The passive reading — the nightly off-site push — is only a low floor (2026-10-24 night: demo-hp „at
|
||
// least 10 Mbit/s"; this test read 90.7 on the same line). So once a week the HOST (the tunnel lives on the host)
|
||
// runs a 10-second iperf3 upload to ep0's tunnel-only iperf3 server (felhom.eu scripts/ep0-iperf3). No file is left
|
||
// anywhere and nothing counts against the household's storage.
|
||
//
|
||
// The rule, each part pinned by a test in linetest_test.go:
|
||
// - Weekly: due from Tuesday 10:00 local until one measurement succeeded since then (a box that was off catches up
|
||
// on a later day, inside the window).
|
||
// - Daytime only: a test starts only 10:00–15:59 local — never in the night chain's window (01:00–06:59), and each
|
||
// host starts from its own minute (SlotMinute, spread over 10:00–13:59) so the fleet does not test at once.
|
||
// - Never beside a backup, a restore-test, an OS update or a trim: the test holds the host-wide one-heavy-op gate.
|
||
// - Only after QuietMinutes of quiet uplink: every one-minute transmit rate of the host's physical uplink below
|
||
// QuietMbps — so neither the family's upload nor a restore is measured against itself.
|
||
// - Up to RunsPerMeasurement runs of TestTime each; the highest counts. A run is bounded by RunTimeout.
|
||
// - „Server busy" (another box is testing) → tried again at the next tick the same day; at most MaxTriesPerDay.
|
||
// - No tunnel (no operational wg-felhom with a server address) → no test and no value; the controller then keeps its
|
||
// passive floor and sets no qBittorrent limit.
|
||
// - The last result is persisted, so a restart neither loses it nor re-tests.
|
||
package linetest
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"hash/fnv"
|
||
"log/slog"
|
||
"net/netip"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||
)
|
||
|
||
const (
|
||
Weekday = time.Tuesday
|
||
StartHour = 10 // first eligible local hour (inclusive)
|
||
EndHour = 16 // first NOT-eligible local hour: the last start is 15:59
|
||
SlotSpreadMinutes = 240
|
||
QuietMinutes = 10
|
||
QuietMbps = 2.0
|
||
RunsPerMeasurement = 3
|
||
TestTime = 10 * time.Second
|
||
Streams = 4
|
||
RunTimeout = 25 * time.Second
|
||
RunGap = 30 * time.Second
|
||
MaxTriesPerDay = 6
|
||
ServerPort = 5201
|
||
TickInterval = time.Minute
|
||
FirstTickDelay = 5 * time.Minute
|
||
GateName = "line-test"
|
||
)
|
||
|
||
// ScheduleText is carried on the host report.
|
||
const ScheduleText = "weekly, due Tuesday from 10:00 host-local time; starts 10:00-15:59 after 10 quiet minutes; never beside a backup, restore-test or update"
|
||
|
||
// Gate is the host-wide one-heavy-operation gate (*backup.InFlight).
|
||
type Gate interface {
|
||
TryAcquire(what string) (release func(), busy string, ok bool)
|
||
}
|
||
|
||
// Target answers the iperf3 server's tunnel address, ok=false when the tunnel is not operational.
|
||
type Target func() (netip.Addr, bool)
|
||
|
||
// Runner runs one upload test (Client in production).
|
||
type Runner interface {
|
||
Run(ctx context.Context, addr string) (Result, error)
|
||
}
|
||
|
||
// UplinkTx reads the host's physical-uplink transmit byte counter.
|
||
type UplinkTx func() (uint64, error)
|
||
|
||
// State is what is persisted (line-test.json).
|
||
type State struct {
|
||
LastOKAt time.Time `json:"last_ok_at,omitempty"`
|
||
Mbps float64 `json:"mbps,omitempty"`
|
||
RunsMbps []float64 `json:"runs_mbps,omitempty"`
|
||
Server string `json:"server,omitempty"`
|
||
LastAttemptAt time.Time `json:"last_attempt_at,omitempty"`
|
||
LastError string `json:"last_error,omitempty"`
|
||
TriesDay string `json:"tries_day,omitempty"` // local date of TriesToday
|
||
TriesToday int `json:"tries_today,omitempty"`
|
||
}
|
||
|
||
type txSample struct {
|
||
at time.Time
|
||
tx uint64
|
||
}
|
||
|
||
// Tester is the weekly job.
|
||
type Tester struct {
|
||
runner Runner
|
||
target Target
|
||
gate Gate
|
||
uplink UplinkTx
|
||
statePath string
|
||
slot int // minutes after StartHour:00 this host may first start
|
||
logger *slog.Logger
|
||
loc *time.Location
|
||
now func() time.Time
|
||
sleep func(context.Context, time.Duration) bool
|
||
|
||
mu sync.Mutex
|
||
st State
|
||
samples []txSample
|
||
}
|
||
|
||
// New builds the job and loads the persisted state (missing = empty; corrupt = logged, empty — the cost is one extra
|
||
// test, never a lost value elsewhere).
|
||
func New(runner Runner, target Target, gate Gate, uplink UplinkTx, hostID, statePath string, logger *slog.Logger) *Tester {
|
||
if logger == nil {
|
||
logger = slog.Default()
|
||
}
|
||
t := &Tester{runner: runner, target: target, gate: gate, uplink: uplink, statePath: statePath,
|
||
slot: SlotMinute(hostID), logger: logger, loc: time.Local, now: time.Now, sleep: sleepCtx}
|
||
t.load()
|
||
return t
|
||
}
|
||
|
||
func sleepCtx(ctx context.Context, d time.Duration) bool {
|
||
tm := time.NewTimer(d)
|
||
defer tm.Stop()
|
||
select {
|
||
case <-ctx.Done():
|
||
return false
|
||
case <-tm.C:
|
||
return true
|
||
}
|
||
}
|
||
|
||
// SlotMinute spreads hosts over the first SlotSpreadMinutes of the window, stable per host id.
|
||
func SlotMinute(hostID string) int {
|
||
h := fnv.New32a()
|
||
h.Write([]byte(hostID))
|
||
return int(h.Sum32() % SlotSpreadMinutes)
|
||
}
|
||
|
||
func (t *Tester) load() {
|
||
data, err := os.ReadFile(t.statePath)
|
||
if err != nil {
|
||
if !os.IsNotExist(err) {
|
||
t.logger.Warn("linetest: state read failed — starting empty", "path", t.statePath, "err", err)
|
||
}
|
||
return
|
||
}
|
||
if err := json.Unmarshal(data, &t.st); err != nil {
|
||
t.logger.Warn("linetest: state file corrupt — starting empty", "path", t.statePath, "err", err)
|
||
t.st = State{}
|
||
}
|
||
}
|
||
|
||
func (t *Tester) saveLocked() error {
|
||
data, err := json.MarshalIndent(t.st, "", " ")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := os.MkdirAll(filepath.Dir(t.statePath), 0o755); err != nil {
|
||
return err
|
||
}
|
||
tmp := t.statePath + ".tmp"
|
||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||
os.Remove(tmp)
|
||
return err
|
||
}
|
||
return os.Rename(tmp, t.statePath)
|
||
}
|
||
|
||
// EligibleAt reports whether a test may START at local time lt for a host whose slot is slot minutes after
|
||
// StartHour:00 on the due day. On a catch-up day (anchorDay=false) the slot is not waited for again.
|
||
func EligibleAt(lt time.Time, slot int, anchorDay bool) bool {
|
||
h := lt.Hour()
|
||
if h < StartHour || h >= EndHour {
|
||
return false
|
||
}
|
||
if anchorDay {
|
||
mins := (h-StartHour)*60 + lt.Minute()
|
||
return mins >= slot
|
||
}
|
||
return true
|
||
}
|
||
|
||
// weekAnchor is the most recent Tuesday StartHour:00 at or before lt.
|
||
func weekAnchor(lt time.Time) time.Time {
|
||
daysBack := (int(lt.Weekday()) - int(Weekday) + 7) % 7
|
||
d := lt.AddDate(0, 0, -daysBack)
|
||
a := time.Date(d.Year(), d.Month(), d.Day(), StartHour, 0, 0, 0, lt.Location())
|
||
if a.After(lt) {
|
||
d = d.AddDate(0, 0, -7)
|
||
a = time.Date(d.Year(), d.Month(), d.Day(), StartHour, 0, 0, 0, lt.Location())
|
||
}
|
||
return a
|
||
}
|
||
|
||
// Due reports whether a measurement is due at lt given the state.
|
||
func Due(st State, lt time.Time) bool {
|
||
return st.LastOKAt.IsZero() || st.LastOKAt.In(lt.Location()).Before(weekAnchor(lt))
|
||
}
|
||
|
||
// sample records the uplink counter (once a minute) and keeps QuietMinutes+1 samples.
|
||
func (t *Tester) sample() {
|
||
if t.uplink == nil {
|
||
return
|
||
}
|
||
tx, err := t.uplink()
|
||
if err != nil {
|
||
t.logger.Debug("linetest: uplink counter unreadable", "err", err)
|
||
t.mu.Lock()
|
||
t.samples = nil // a gap is not quiet
|
||
t.mu.Unlock()
|
||
return
|
||
}
|
||
t.mu.Lock()
|
||
t.samples = append(t.samples, txSample{at: t.now(), tx: tx})
|
||
if len(t.samples) > QuietMinutes+1 {
|
||
t.samples = t.samples[len(t.samples)-(QuietMinutes+1):]
|
||
}
|
||
t.mu.Unlock()
|
||
}
|
||
|
||
// quiet reports whether the last QuietMinutes minutes each sent below QuietMbps, and the highest minute seen.
|
||
func quiet(samples []txSample) (bool, float64) {
|
||
if len(samples) < QuietMinutes+1 {
|
||
return false, 0
|
||
}
|
||
worst := 0.0
|
||
for i := 1; i < len(samples); i++ {
|
||
dt := samples[i].at.Sub(samples[i-1].at).Seconds()
|
||
if dt <= 0 || dt > 3*TickInterval.Seconds() || samples[i].tx < samples[i-1].tx {
|
||
return false, worst // a gap or a counter reset is not proof of quiet
|
||
}
|
||
mbps := float64(samples[i].tx-samples[i-1].tx) * 8 / dt / 1e6
|
||
if mbps > worst {
|
||
worst = mbps
|
||
}
|
||
}
|
||
return worst < QuietMbps, worst
|
||
}
|
||
|
||
// Run ticks every TickInterval until ctx ends. It never returns an error: a failed test is a reported fact.
|
||
func (t *Tester) Run(ctx context.Context) {
|
||
t.logger.Info("linetest: weekly upload line test starting", "schedule", ScheduleText, "slot_minute", t.slot)
|
||
if !t.sleep(ctx, FirstTickDelay) {
|
||
return
|
||
}
|
||
for {
|
||
t.sample()
|
||
t.Tick(ctx)
|
||
if !t.sleep(ctx, TickInterval) {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// Tick is one look.
|
||
func (t *Tester) Tick(ctx context.Context) {
|
||
lt := t.now().In(t.loc)
|
||
t.mu.Lock()
|
||
st := t.st
|
||
samples := append([]txSample(nil), t.samples...)
|
||
t.mu.Unlock()
|
||
if !Due(st, lt) {
|
||
return
|
||
}
|
||
anchor := weekAnchor(lt)
|
||
anchorDay := lt.YearDay() == anchor.YearDay() && lt.Year() == anchor.Year()
|
||
if !EligibleAt(lt, t.slot, anchorDay) {
|
||
return
|
||
}
|
||
day := lt.Format("2006-01-02")
|
||
if st.TriesDay == day && st.TriesToday >= MaxTriesPerDay {
|
||
return
|
||
}
|
||
ip, ok := t.target()
|
||
if !ok {
|
||
t.logger.Debug("linetest: no operational tunnel — no test")
|
||
return
|
||
}
|
||
if q, worst := quiet(samples); !q {
|
||
t.logger.Debug("linetest: uplink not quiet — waiting", "worst_minute_mbps", fmt.Sprintf("%.1f", worst), "samples", len(samples))
|
||
return
|
||
}
|
||
release, busy, ok := t.gate.TryAcquire(GateName)
|
||
if !ok {
|
||
t.logger.Info("linetest: deferred — a heavy operation is in flight", "busy", busy)
|
||
return
|
||
}
|
||
defer release()
|
||
t.measure(ctx, netip.AddrPortFrom(ip, ServerPort).String(), day)
|
||
}
|
||
|
||
func (t *Tester) measure(ctx context.Context, addr, day string) {
|
||
start := t.now()
|
||
var runs []float64
|
||
var lastErr error
|
||
busy := false
|
||
for i := 0; i < RunsPerMeasurement; i++ {
|
||
if i > 0 && !t.sleep(ctx, RunGap) {
|
||
break
|
||
}
|
||
rctx, cancel := context.WithTimeout(ctx, RunTimeout)
|
||
r, err := t.runner.Run(rctx, addr)
|
||
cancel()
|
||
if errors.Is(err, ErrBusy) {
|
||
busy, lastErr = true, err
|
||
break
|
||
}
|
||
if err != nil {
|
||
lastErr = err
|
||
t.logger.Warn("linetest: run failed", "run", i+1, "server", addr, "err", err)
|
||
continue
|
||
}
|
||
runs = append(runs, r.Mbps)
|
||
t.logger.Info(fmt.Sprintf("linetest: run %d: %.1f Mbit/s", i+1, r.Mbps), "server", addr,
|
||
"server_bytes", r.ServerBytes, "server_seconds", r.ServerSeconds)
|
||
}
|
||
|
||
t.mu.Lock()
|
||
defer t.mu.Unlock()
|
||
if t.st.TriesDay != day {
|
||
t.st.TriesDay, t.st.TriesToday = day, 0
|
||
}
|
||
t.st.TriesToday++
|
||
t.st.LastAttemptAt = start.UTC()
|
||
if len(runs) > 0 {
|
||
best := runs[0]
|
||
for _, r := range runs {
|
||
if r > best {
|
||
best = r
|
||
}
|
||
}
|
||
t.st.LastOKAt, t.st.Mbps, t.st.RunsMbps, t.st.Server, t.st.LastError = start.UTC(), best, runs, addr, ""
|
||
t.logger.Info(fmt.Sprintf("linetest: upload line measured: %.1f Mbit/s (best of %d)", best, len(runs)),
|
||
"server", addr, "runs", len(runs), "duration_s", t.now().Sub(start).Seconds())
|
||
} else {
|
||
msg := "no run succeeded"
|
||
if lastErr != nil {
|
||
msg = lastErr.Error()
|
||
}
|
||
if len(msg) > 300 {
|
||
msg = msg[:300]
|
||
}
|
||
t.st.LastError = msg
|
||
if busy {
|
||
t.logger.Info("linetest: the test server is busy — trying again later today", "server", addr, "tries_today", t.st.TriesToday)
|
||
} else {
|
||
t.logger.Warn("linetest: measurement failed", "server", addr, "tries_today", t.st.TriesToday, "err", msg)
|
||
}
|
||
}
|
||
if err := t.saveLocked(); err != nil {
|
||
t.logger.Warn("linetest: state save failed — the result will not survive a restart", "path", t.statePath, "err", err)
|
||
}
|
||
}
|
||
|
||
// Value is the measured line (ok=false: never measured).
|
||
func (t *Tester) Value() (mbps float64, at time.Time, ok bool) {
|
||
t.mu.Lock()
|
||
defer t.mu.Unlock()
|
||
if t.st.LastOKAt.IsZero() || t.st.Mbps <= 0 {
|
||
return 0, time.Time{}, false
|
||
}
|
||
return t.st.Mbps, t.st.LastOKAt, true
|
||
}
|
||
|
||
// LineTestStatus implements hub.LineTestReporter: a pure read of the persisted state.
|
||
func (t *Tester) LineTestStatus(context.Context) *hub.LineTestStatus {
|
||
t.mu.Lock()
|
||
defer t.mu.Unlock()
|
||
out := &hub.LineTestStatus{Schedule: ScheduleText, Mbps: t.st.Mbps, RunsMbps: t.st.RunsMbps, Server: t.st.Server,
|
||
LastError: t.st.LastError}
|
||
if !t.st.LastOKAt.IsZero() {
|
||
out.LastOKAt = t.st.LastOKAt.UTC().Format(time.RFC3339)
|
||
}
|
||
if !t.st.LastAttemptAt.IsZero() {
|
||
out.LastAttemptAt = t.st.LastAttemptAt.UTC().Format(time.RFC3339)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ── the host's physical uplink counter ──────────────────────────────────────────────────────────────────────────
|
||
|
||
// SysUplinkTx reads the transmit bytes of the host's physical uplink under root (normally "/"): the interface of the
|
||
// default route in /proc/net/route, and — when it is a bridge (vmbr0), as on every Proxmox host — the sum of its
|
||
// PHYSICAL member ports (those with a `device` link). The bridge's own counter does not see the guests' bridged
|
||
// traffic, which is the traffic this check exists for.
|
||
func SysUplinkTx(root string) UplinkTx {
|
||
return func() (uint64, error) {
|
||
ifc, err := defaultRouteIface(filepath.Join(root, "proc/net/route"))
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
sys := filepath.Join(root, "sys/class/net")
|
||
ports := []string{ifc}
|
||
if ents, err := os.ReadDir(filepath.Join(sys, ifc, "brif")); err == nil {
|
||
ports = ports[:0]
|
||
for _, e := range ents {
|
||
if _, err := os.Stat(filepath.Join(sys, e.Name(), "device")); err == nil {
|
||
ports = append(ports, e.Name())
|
||
}
|
||
}
|
||
if len(ports) == 0 {
|
||
return 0, fmt.Errorf("linetest: bridge %s has no physical port", ifc)
|
||
}
|
||
}
|
||
var sum uint64
|
||
for _, p := range ports {
|
||
b, err := os.ReadFile(filepath.Join(sys, p, "statistics/tx_bytes"))
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
n, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
sum += n
|
||
}
|
||
return sum, nil
|
||
}
|
||
}
|
||
|
||
func defaultRouteIface(path string) (string, error) {
|
||
b, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
for _, line := range strings.Split(string(b), "\n")[1:] {
|
||
f := strings.Fields(line)
|
||
if len(f) >= 8 && f[1] == "00000000" && f[7] == "00000000" {
|
||
return f[0], nil
|
||
}
|
||
}
|
||
return "", errors.New("linetest: no default route")
|
||
}
|