From aa4dfb75ea3001e21f56000f4f711088999be6c2 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 10 Jun 2026 16:16:03 +0200 Subject: [PATCH] slice 9: GET /host/metrics + CPU/chassis-temp collector (v0.14.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a host-wide, token-authed GET /host/metrics local-API endpoint that re-serves the slice-4 collector's host + per-storage view to the customer (the de-privileged controller can't read the host itself). Add the one new collector — CPU/chassis temperature via sysfs hwmon/thermal-zones, graceful- null — to the shared HostMetrics struct, so the hub report carries cpu_temp_c too. Cross-repo host-report golden updated byte-identical. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 15 ++ REPORT.md | 86 ++++----- cmd/felhom-agent/main.go | 11 +- internal/hub/collect.go | 36 +++- internal/hub/cputemp.go | 179 ++++++++++++++++++ internal/hub/cputemp_test.go | 89 +++++++++ internal/hub/hostmetrics_test.go | 71 +++++++ internal/hub/report.go | 6 + internal/hub/report_test.go | 3 +- internal/hub/testdata/host-report.golden.json | 3 +- internal/localapi/host_metrics.go | 49 +++++ internal/localapi/host_metrics_test.go | 144 ++++++++++++++ internal/localapi/server.go | 27 ++- 13 files changed, 664 insertions(+), 55 deletions(-) create mode 100644 internal/hub/cputemp.go create mode 100644 internal/hub/cputemp_test.go create mode 100644 internal/hub/hostmetrics_test.go create mode 100644 internal/localapi/host_metrics.go create mode 100644 internal/localapi/host_metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b094021..f0c77e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ All notable changes to **felhom-agent** are recorded here. Update on every code change that gets pushed. +## v0.14.0 — slice 9: host metrics to the controller (`GET /host/metrics` + CPU-temp collector) (2026-06-10) + +The de-privileged controller (slice 8C) sees only its own cgroup, so it can't read host health itself. Slice 9 **re-serves** the slice-4 collector's host + per-storage view to the customer over the local API, plus the one missing collector — CPU/chassis temperature — so the customer sees their box's health in the controller. Host-wide, token-authed, fresh (a live collect, not the 15-min hub snapshot). Assumption: **one customer per host** (the home-server model); if a host ever serves multiple customers, host-wide CPU/mem would leak cross-customer load → revisit then. + +### Added / changed +- **CPU/chassis-temp collector** (`internal/hub/cputemp.go`): `SysfsTempReader` reads the CPU package temperature straight from sysfs — hwmon (`coretemp`/`k10temp`/`zenpower`/`cpu_thermal`, preferring the `Package id 0` input) then the thermal zones (preferring `x86_pkg_temp`/`coretemp`/`cpu-thermal`, falling back to `acpitz`). **No external binary, no privilege** (sysfs nodes are world-readable), so the root-CLI fence is untouched. **Graceful-null**: a missing sensor, an unsupported board, an implausible reading (outside 5–150 °C), or any read error all degrade to `null` ("n/a") — a missing sensor never fails the report. Wired into the collector via the new `TempReader` seam (nil-safe). +- **`HostMetrics.CPUTempC *int` (`cpu_temp_c`)** — new nullable wire field on the **shared** `HostMetrics` struct (same nullable contract as the disk `SmartSummary.TemperatureC`). It rides the **hub report too** (operator freebie) → cross-repo host-report golden updated. +- **`Collector.HostMetricsNow(ctx)`** — a fresh `NodeStatus` + CPU-temp read returning just the host block, the source for the local API (current cpu%/temp, not the 15-min snapshot). `Collect()` now also populates `cpu_temp_c` on the hub report. `Collector.SetTempReader` injects a fake in tests. +- **`GET /host/metrics`** (`internal/localapi/host_metrics.go`): host-wide health (cpu%/mem/load/uptime/`cpu_temp_c`) + per-storage capacity targets (total/used/fraction, thin-pool, SMART temp+wear). Token-authed via `withGuest` (host-wide data; cross-guest `?vmid=` still 403). Best-effort on storage (a view error still returns the host block). Served only when the `HostMetrics` provider (the shared collector) is wired — else 503 "not configured". Wired in `buildLocalAPIServer`. + +### Tests +- `cputemp_test.go`: a fake `/sys` layout proves hwmon package-preference, hwmon first-input fallback, thermal-zone-by-type selection over a non-CPU hwmon, **graceful-null on a sensorless host** (no error), and rejection of implausible (0 m°C) readings. +- `hostmetrics_test.go`: `HostMetricsNow` populates the temp, gracefully nulls it, hard-errors on `NodeStatus` failure; `Collect()` carries the temp. +- `host_metrics_test.go` (localapi): populated host+storage with a valid token; `cpu_temp_c:null` serializes; **401 without a token** (collector never invoked); 403 on a cross-guest `?vmid=`; 503 when not configured. + ## v0.13.0 — slice 8B.2: quiesce downtime optimization (`snapshotted` phase) (2026-06-10) The agent half of slice 8B.2. In snapshot mode, vzdump only needs the app-stopped state captured at diff --git a/REPORT.md b/REPORT.md index 8209a1b..fe490a2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,53 +1,53 @@ -# REPORT — slice 8B.2 (agent half): quiesce downtime optimization — `snapshotted` phase (v0.13.0) (2026-06-10) +# REPORT — slice 9 (agent half): host metrics to the controller (v0.14.0) (2026-06-10) -> Overwrite-latest report. Cumulative history: [CHANGELOG.md](CHANGELOG.md). Implements the agent -> half of `TASK — Slice 8B.2`. The controller early-resume is `felhom-controller` v0.38.0. No hub change. +> Overwrite-latest report. Cumulative history: [CHANGELOG.md](CHANGELOG.md). -## Outcome +## What was implemented -In snapshot mode an LXC vzdump only needs the app-stopped state captured at the **storage-snapshot -moment**; after that it reads from the snapshot and the app can safely resume. The agent now emits a -**`snapshotted`** phase on `GET /backup/status` when the snapshot is taken, so the controller resumes -its app early — app downtime drops from *whole-backup* to *until-snapshot* with no loss of -app-consistency. +The agent half of **slice 9** — re-serving the host's health to the customer's controller, plus the +one new collector (CPU/chassis temperature). The de-privileged controller (slice 8C) sees only its +own cgroup, so it cannot read host metrics itself; the agent already collects host CPU/mem/loadavg/ +uptime + per-storage targets for the hub, and slice 9 exposes that to the customer over the local API. -## Phase 0 (validated first, on the demo — PVE 9.2.2) +### `internal/hub/cputemp.go` — CPU/chassis-temp collector (new) +- `TempReader` seam + `SysfsTempReader`: reads the CPU package temperature from **sysfs** — hwmon + (`coretemp`/`k10temp`/`zenpower`/`cpu_thermal`, preferring the `Package id 0` input) then thermal + zones (preferring `x86_pkg_temp`/`coretemp`/`cpu-thermal`, falling back to `acpitz`). +- **No external binary, no privilege** (sysfs is world-readable) → the root-CLI fence is untouched. +- **Graceful-null**: a missing sensor, an unsupported board, an implausible reading (outside + 5–150 °C), or any read error all degrade to `nil` ("n/a") — never fails the report. Same nullable + contract as the per-disk `SmartSummary.TemperatureC`. -- **Snapshot mode is in effect** on local-lvm (lvmthin) — `vzdump --mode snapshot` logs `backup mode: - snapshot`, not downgraded to stop. -- **Marker:** `INFO: create storage snapshot 'vzdump'` — the vzdump-level step after which the backup - reads from the snapshot (generic across snapshot-capable storage; only appears in snapshot mode). -- **Downtime delta:** snapshot created in the first ~1s; archive runs ~23s after. So 8B (resume at - `done`) = ~24s app-down; 8B.2 (resume at `snapshotted`) = ~1s — **~95% cut** for a 934 MB guest; - the delta grows with guest size. +### `internal/hub` — shared wire field + collector reuse +- `HostMetrics` gains **`CPUTempC *int` (`cpu_temp_c`)** — nullable, on the **shared** struct, so the + **hub report carries it too** (operator freebie). Cross-repo host-report golden updated + **byte-identical** with the hub's copy. +- `Collector` gains a nil-safe `temp TempReader` (defaults to the real `SysfsTempReader`; + `SetTempReader` injects a fake in tests). `Collect()` now sets `cpu_temp_c` on the report. +- **`Collector.HostMetricsNow(ctx)`** — a fresh `NodeStatus` + CPU-temp read returning just the host + block; the source for the local API (current cpu%/temp, not the 15-min hub snapshot). -## What landed +### `internal/localapi` — `GET /host/metrics` (new endpoint) +- `host_metrics.go`: host-wide health (cpu%/mem/load/uptime/`cpu_temp_c`) + per-storage capacity + (total/used/fraction, thin-pool, SMART temp+wear). Token-authed via `withGuest` (host-wide data; a + cross-guest `?vmid=` still 403). Best-effort on storage (a view error still returns the host + block). Served only when the `HostMetrics` provider (the shared collector) is wired in + `buildLocalAPIServer` — else 503 "not configured". -- **`BackupRunner.BackupWithSnapshotHook(ctx, vmid, onSnapshot)`** — while the vzdump runs, a watcher - tails the task log (`TaskLogTail`) for the `create storage snapshot` marker and fires `onSnapshot` - **once**. It bails on `backup mode: stop` and the marker never appears in stop mode, so it **never - fires in stop/downgraded mode**. `Backup` keeps its signature (scheduler/selftest); both share one - body. `snapshotWatchInterval` is a package var (prod 1s; tests shrink it). -- **`/backup/status` phase `snapshotted`** (between `running` and `done`): `handleBackup` passes the - hook → `markSnapshotted` flips the running job. `done`/`failed` unchanged; `snapshotted` is additive. +## Tests (all green) +- `cputemp_test.go`, `hostmetrics_test.go`, `host_metrics_test.go`: hwmon/thermal-zone selection + + **graceful-null**, `HostMetricsNow` populate/null/hard-error, endpoint populated + `cpu_temp_c:null` + serialization + **401 without a token** + 403 cross-guest + 503 not-configured. +- `go test ./...` green; `go vet ./internal/hub ./internal/localapi` clean. -## Tests +## Versioning / docs +- Version `0.13.0 → 0.14.0`; `CHANGELOG.md` updated. Doc 03 §6 (local-API surface) + §9 (roadmap + + changelog) updated. -`go test ./...` green; `-race` green (build server). localapi: snapshot mode → `snapshotted` before -`done`; stop mode → `snapshotted` never emitted. runner: the watcher fires on the marker; stop-mode -log never fires. +## Assumption (noted, not built) +- **One customer per host** (home-server model): `/host/metrics` is host-wide. A multi-customer host + would leak cross-customer CPU/mem → revisit then. -## Live validation (demo-felhom) - -End-to-end on a provisioned guest (controller v0.38.0 + a postgres stack): the agent logged `backup -reached snapshotted (app may resume)` mid-backup, the controller resumed the app at that point, and -the backup proceeded to `done`. **App downtime ~3s** (quiesce→snapshotted) vs **~23s** if it had -waited for `done` (~87% cut). The snapshot backup restored **clean** (`database system was shut down`, -no WAL replay) — the early resume did not compromise app-consistency. Deployed to the demo service as -v0.13.0. - -## Deferred / dependency - -Snapshot-capable storage (lvm-thin/ZFS) is required for the win; on stop/downgraded storage the -controller falls back to resume-at-`done` (8B). No consistency-contract or crash-safety change. No -secrets committed. +## Pending +- **Live validation** on the demo (build + deploy agent v0.14.0; controller monitoring page → real + N100 CPU%/temp + per-storage, cross-checked vs `pvesh`/`free`/`df`). diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 9a60369..4490ae6 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -40,7 +40,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.13.0" +var version = "0.14.0" func main() { var ( @@ -337,7 +337,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { // host still reports/reconciles). The leaf is generated+persisted once so its pin is stable. localServers := 0 var localTokens *localapi.TokenStore - localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, logger, &localTokens) + localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, collector, logger, &localTokens) if localTokens != nil { defer localTokens.Close() } @@ -462,7 +462,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re // leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the // daemon — the host still reports/reconciles; only the controller channel is unavailable until // fixed. The opened token store is returned via outTokens so the caller can Close it. -func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps storage.HostOps, gate *reconcile.Gate, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { +func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps storage.HostOps, gate *reconcile.Gate, collector *hub.Collector, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { if !cfg.LocalAPI.Enabled() { return nil } @@ -497,7 +497,10 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St Disks: hostOps, DiskGate: storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID}, Guests2: px, - Logger: logger, + // Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host + + // per-storage view to the customer's monitoring page (reuses the slice-4 collector). + HostMetrics: collector, + Logger: logger, }) if err != nil { logger.Warn("daemon: local-api disabled (server build)", "err", err) diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 10eee71..51ce70f 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -57,6 +57,7 @@ type Collector struct { backups BackupReporter restoreTests RestoreTestReporter pbs PBSReporter + temp TempReader // slice 9: host CPU/chassis temp (nil-safe → nil temp) hostID string agentVersion string logger *slog.Logger @@ -76,6 +77,7 @@ func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserve backups: backups, restoreTests: restoreTests, pbs: pbs, + temp: SysfsTempReader{}, // slice 9: real sysfs reader by default; tests inject a fake hostID: hostID, agentVersion: agentVersion, logger: logger, @@ -83,6 +85,13 @@ func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserve } } +// SetTempReader overrides the host-temp source (tests inject a fake; a nil reader disables temp). +// Returns the collector for chaining. +func (c *Collector) SetTempReader(t TempReader) *Collector { + c.temp = t + return c +} + // Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard // error (no useful report — the cycle skips the POST); a failed per-guest // GuestConfig degrades that guest to status="unknown" without spec but still sends; @@ -93,11 +102,13 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { return nil, fmt.Errorf("hub: NodeStatus failed (no useful report): %w", err) } + host := hostMetrics(c.px.Node(), ns) + host.CPUTempC = c.cpuTempC(ctx) // slice 9: operator freebie — temp now rides the hub report too report := &HostReport{ HostID: c.hostID, ReportedAt: c.now().Format(time.RFC3339), AgentVersion: c.agentVersion, - Host: hostMetrics(c.px.Node(), ns), + Host: host, Guests: c.collectGuests(ctx), // storage_targets populated this slice (slice 5) via the observer; the rest stay // defined-but-empty (slice 6). Non-nil so they marshal as []. @@ -112,6 +123,29 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { return report, nil } +// HostMetricsNow does a FRESH NodeStatus + CPU-temp read and returns just the host block (no +// guests/storage). It is the source for the local API's GET /host/metrics (slice 9) — current +// cpu%/temp, not the 15-min hub-report snapshot. Storage targets come from the observer +// separately. A NodeStatus failure is a hard error (no useful host view); a missing temp sensor +// degrades to nil (never an error). +func (c *Collector) HostMetricsNow(ctx context.Context) (HostMetrics, error) { + ns, err := c.px.NodeStatus(ctx) + if err != nil { + return HostMetrics{}, fmt.Errorf("hub: NodeStatus failed: %w", err) + } + h := hostMetrics(c.px.Node(), ns) + h.CPUTempC = c.cpuTempC(ctx) + return h, nil +} + +// cpuTempC reads the host CPU/chassis temp via the TempReader seam (nil-safe → nil). +func (c *Collector) cpuTempC(ctx context.Context) *int { + if c.temp == nil { + return nil + } + return c.temp.CPUTempC(ctx) +} + func hostMetrics(node string, ns proxmox.NodeStatus) HostMetrics { h := HostMetrics{ Node: node, diff --git a/internal/hub/cputemp.go b/internal/hub/cputemp.go new file mode 100644 index 0000000..28d5b6c --- /dev/null +++ b/internal/hub/cputemp.go @@ -0,0 +1,179 @@ +package hub + +import ( + "context" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// TempReader reads the host CPU/chassis temperature in whole °C, returning nil when no usable +// sensor is exposed. It is the slice-9 collector seam — graceful-null is the contract: a missing +// sensor, an unsupported board, or any read error all degrade to nil rather than failing the +// host report (mirrors the nullable disk SmartSummary.TemperatureC). The collector is nil-safe +// (a nil TempReader yields nil temp). +type TempReader interface { + CPUTempC(ctx context.Context) *int +} + +// SysfsTempReader reads the CPU package temperature straight from sysfs (hwmon coretemp/k10temp/ +// cpu_thermal, then the thermal zones). No external binary and no privilege is needed — these +// nodes are world-readable — so it never shells out (keeping the agent's root-CLI fence intact). +// It prefers the CPU-package hwmon sensor; only if hwmon yields nothing does it fall back to a +// CPU-ish thermal zone. Every failure path returns nil ("n/a"). +type SysfsTempReader struct { + // Root overrides the sysfs root ("" → "/sys"); set by tests to a fake layout. + Root string +} + +func (r SysfsTempReader) root() string { + if r.Root != "" { + return r.Root + } + return "/sys" +} + +// CPUTempC returns the CPU/chassis temperature in whole °C, or nil if nothing usable is exposed. +// ctx is accepted for interface symmetry (the reads are local sysfs and effectively instant). +func (r SysfsTempReader) CPUTempC(ctx context.Context) *int { + if t := r.fromHwmon(); t != nil { + return t + } + if t := r.fromThermalZones(); t != nil { + return t + } + return nil +} + +// cpuHwmonNames are the kernel hwmon driver names that expose a CPU temperature: coretemp +// (Intel — e.g. the demo N100), k10temp/zenpower (AMD), cpu_thermal (ARM SoCs). +var cpuHwmonNames = map[string]bool{ + "coretemp": true, + "k10temp": true, + "zenpower": true, + "cpu_thermal": true, +} + +// fromHwmon scans /sys/class/hwmon/hwmon*/ for a CPU driver and returns its package temperature. +// For a multi-core coretemp it prefers the "Package id 0" labelled input; otherwise it takes the +// first readable tempN_input. Returns nil when no CPU hwmon is present/readable. +func (r SysfsTempReader) fromHwmon() *int { + dirs, err := filepath.Glob(filepath.Join(r.root(), "class", "hwmon", "hwmon*")) + if err != nil { + return nil + } + sort.Strings(dirs) // deterministic device ordering (hwmon0, hwmon1, …) + for _, dir := range dirs { + name := strings.TrimSpace(readFileTrim(filepath.Join(dir, "name"))) + if !cpuHwmonNames[name] { + continue + } + if t := readHwmonPackageTemp(dir); t != nil { + return t + } + } + return nil +} + +// readHwmonPackageTemp returns a CPU hwmon device's package temperature, preferring a +// tempN_input whose tempN_label is "Package id 0", else the lowest-numbered readable input. +func readHwmonPackageTemp(dir string) *int { + inputs, err := filepath.Glob(filepath.Join(dir, "temp*_input")) + if err != nil || len(inputs) == 0 { + return nil + } + sort.Strings(inputs) // temp1_input < temp10_input lexically is wrong, but the package is temp1 + var firstReadable *int + for _, in := range inputs { + milli, ok := readMilliC(in) + if !ok { + continue + } + c := milli / 1000 + if firstReadable == nil { + v := c + firstReadable = &v + } + labelPath := strings.TrimSuffix(in, "_input") + "_label" + if strings.EqualFold(strings.TrimSpace(readFileTrim(labelPath)), "Package id 0") { + v := c + return &v + } + } + return firstReadable +} + +// cpuZoneTypes are thermal-zone `type` values that name a CPU sensor, in preference order. +var cpuZoneTypes = []string{"x86_pkg_temp", "coretemp", "cpu-thermal", "cpu_thermal", "soc_thermal"} + +// fromThermalZones scans /sys/class/thermal/thermal_zone*/ and returns the best CPU-ish zone's +// temperature. It prefers a zone whose `type` matches a known CPU sensor (in cpuZoneTypes order); +// if none match it falls back to an acpitz zone, then the first readable zone. nil when none read. +func (r SysfsTempReader) fromThermalZones() *int { + zones, err := filepath.Glob(filepath.Join(r.root(), "class", "thermal", "thermal_zone*")) + if err != nil { + return nil + } + sort.Strings(zones) + byType := map[string]*int{} + var acpitz, firstAny *int + for _, z := range zones { + zType := strings.TrimSpace(readFileTrim(filepath.Join(z, "type"))) + milli, ok := readMilliC(filepath.Join(z, "temp")) + if !ok { + continue + } + c := milli / 1000 + if firstAny == nil { + v := c + firstAny = &v + } + if zType == "acpitz" && acpitz == nil { + v := c + acpitz = &v + } + if _, seen := byType[zType]; !seen { + v := c + byType[zType] = &v + } + } + for _, want := range cpuZoneTypes { + if t := byType[want]; t != nil { + return t + } + } + if acpitz != nil { + return acpitz + } + return firstAny +} + +// readMilliC reads a sysfs temperature file (millidegrees Celsius as an integer) and returns it. +// A sane sanity bound rejects obviously bogus values (sensors occasionally report 0 or huge +// numbers when not yet initialised) so "n/a" is reported instead of a garbage temperature. +func readMilliC(path string) (int, bool) { + s := readFileTrim(path) + if s == "" { + return 0, false + } + milli, err := strconv.Atoi(s) + if err != nil { + return 0, false + } + // Plausible CPU/chassis range: 5°C..150°C. Outside that → treat as unavailable. + if milli < 5000 || milli > 150000 { + return 0, false + } + return milli, true +} + +// readFileTrim reads a small sysfs file and trims it; "" on any error (graceful-null). +func readFileTrim(path string) string { + b, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} diff --git a/internal/hub/cputemp_test.go b/internal/hub/cputemp_test.go new file mode 100644 index 0000000..e9abd6f --- /dev/null +++ b/internal/hub/cputemp_test.go @@ -0,0 +1,89 @@ +package hub + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// writeSysfs creates path under root with the given content (sysfs files are tiny text files). +func writeSysfs(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// A coretemp hwmon with a "Package id 0" label must win over a per-core sensor. +func TestSysfsTempReader_HwmonPackagePreferred(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "class", "hwmon", "hwmon0") + writeSysfs(t, filepath.Join(dir, "name"), "coretemp\n") + // temp1 = Package id 0 = 47°C; temp2 = Core 0 = 52°C. The package must be chosen. + writeSysfs(t, filepath.Join(dir, "temp1_input"), "47000\n") + writeSysfs(t, filepath.Join(dir, "temp1_label"), "Package id 0\n") + writeSysfs(t, filepath.Join(dir, "temp2_input"), "52000\n") + writeSysfs(t, filepath.Join(dir, "temp2_label"), "Core 0\n") + + got := SysfsTempReader{Root: root}.CPUTempC(context.Background()) + if got == nil || *got != 47 { + t.Fatalf("CPUTempC = %v, want 47", got) + } +} + +// With no package label, the first readable hwmon input is used. +func TestSysfsTempReader_HwmonFirstInputFallback(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "class", "hwmon", "hwmon0") + writeSysfs(t, filepath.Join(dir, "name"), "k10temp\n") + writeSysfs(t, filepath.Join(dir, "temp1_input"), "39000\n") // Tctl, no label + got := SysfsTempReader{Root: root}.CPUTempC(context.Background()) + if got == nil || *got != 39 { + t.Fatalf("CPUTempC = %v, want 39", got) + } +} + +// A non-CPU hwmon (e.g. a NIC) must be ignored; the thermal-zone CPU sensor is used instead. +func TestSysfsTempReader_ThermalZoneByType(t *testing.T) { + root := t.TempDir() + // hwmon is a non-CPU driver → skipped. + nic := filepath.Join(root, "class", "hwmon", "hwmon0") + writeSysfs(t, filepath.Join(nic, "name"), "iwlwifi\n") + writeSysfs(t, filepath.Join(nic, "temp1_input"), "60000\n") + // thermal zones: acpitz (40°C) + x86_pkg_temp (55°C). The CPU package type must win. + z0 := filepath.Join(root, "class", "thermal", "thermal_zone0") + writeSysfs(t, filepath.Join(z0, "type"), "acpitz\n") + writeSysfs(t, filepath.Join(z0, "temp"), "40000\n") + z1 := filepath.Join(root, "class", "thermal", "thermal_zone1") + writeSysfs(t, filepath.Join(z1, "type"), "x86_pkg_temp\n") + writeSysfs(t, filepath.Join(z1, "temp"), "55000\n") + + got := SysfsTempReader{Root: root}.CPUTempC(context.Background()) + if got == nil || *got != 55 { + t.Fatalf("CPUTempC = %v, want 55 (x86_pkg_temp), got %v", got, got) + } +} + +// The headline graceful-null case: a host that exposes NO sensor (empty /sys) returns nil, and +// no error propagates (CPUTempC has no error return — a missing sensor is "n/a", never a failure). +func TestSysfsTempReader_GracefulNullWhenAbsent(t *testing.T) { + root := t.TempDir() // empty: no hwmon, no thermal zones + if got := (SysfsTempReader{Root: root}).CPUTempC(context.Background()); got != nil { + t.Fatalf("CPUTempC on a sensorless host = %v, want nil", got) + } +} + +// Out-of-range / garbage readings degrade to nil rather than reporting a bogus temperature. +func TestSysfsTempReader_RejectsImplausibleValues(t *testing.T) { + root := t.TempDir() + z := filepath.Join(root, "class", "thermal", "thermal_zone0") + writeSysfs(t, filepath.Join(z, "type"), "x86_pkg_temp\n") + writeSysfs(t, filepath.Join(z, "temp"), "0\n") // 0 m°C → implausible → ignored + if got := (SysfsTempReader{Root: root}).CPUTempC(context.Background()); got != nil { + t.Fatalf("CPUTempC on a 0°C reading = %v, want nil", got) + } +} diff --git a/internal/hub/hostmetrics_test.go b/internal/hub/hostmetrics_test.go new file mode 100644 index 0000000..63d0f21 --- /dev/null +++ b/internal/hub/hostmetrics_test.go @@ -0,0 +1,71 @@ +package hub + +import ( + "context" + "errors" + "testing" +) + +// fakeTemp is a TempReader returning a fixed (nullable) value. +type fakeTemp struct{ c *int } + +func (f fakeTemp) CPUTempC(context.Context) *int { return f.c } + +func intp(v int) *int { return &v } + +// HostMetricsNow returns a fresh host block with cpu% from NodeStatus and the temp from the reader. +func TestHostMetricsNow_PopulatesTemp(t *testing.T) { + px := &fakePx{node: "demo-felhom", ns: newTestNodeStatus()} + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()). + SetTempReader(fakeTemp{c: intp(46)}) + h, err := c.HostMetricsNow(context.Background()) + if err != nil { + t.Fatalf("HostMetricsNow: %v", err) + } + if h.Node != "demo-felhom" || h.CPUPercent != 5 { + t.Errorf("host = %+v", h) + } + if h.CPUTempC == nil || *h.CPUTempC != 46 { + t.Fatalf("cpu_temp_c = %v, want 46", h.CPUTempC) + } + if h.MemoryPercent != 25 { + t.Errorf("mem%% = %v, want 25", h.MemoryPercent) + } +} + +// A missing temp sensor gracefully nulls cpu_temp_c without failing the host read. +func TestHostMetricsNow_GracefulNullTemp(t *testing.T) { + px := &fakePx{node: "n", ns: newTestNodeStatus()} + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()). + SetTempReader(fakeTemp{c: nil}) + h, err := c.HostMetricsNow(context.Background()) + if err != nil { + t.Fatalf("HostMetricsNow: %v", err) + } + if h.CPUTempC != nil { + t.Fatalf("cpu_temp_c = %v, want nil (n/a)", h.CPUTempC) + } +} + +// A NodeStatus failure is a hard error (no useful host view). +func TestHostMetricsNow_NodeStatusErrorIsHard(t *testing.T) { + px := &fakePx{node: "n", nsErr: errors.New("proxmox down")} + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()) + if _, err := c.HostMetricsNow(context.Background()); err == nil { + t.Fatal("NodeStatus failure must be a hard error") + } +} + +// Collect() (the hub report) also carries the temp now — the operator freebie. +func TestCollect_HostReportCarriesTemp(t *testing.T) { + px := &fakePx{node: "n", ns: newTestNodeStatus()} + c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()). + SetTempReader(fakeTemp{c: intp(51)}) + r, err := c.Collect(context.Background()) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if r.Host.CPUTempC == nil || *r.Host.CPUTempC != 51 { + t.Fatalf("report host cpu_temp_c = %v, want 51", r.Host.CPUTempC) + } +} diff --git a/internal/hub/report.go b/internal/hub/report.go index d3fd01d..c6106ac 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -38,6 +38,12 @@ type HostMetrics struct { DiskPercent float64 `json:"disk_percent"` LoadAvg []string `json:"loadavg"` // array of STRINGS (PVE shape) UptimeSeconds int64 `json:"uptime_seconds"` + // CPUTempC is the host CPU/chassis temperature in whole °C, or null when the hardware + // exposes no usable sensor (a headless VM, an unsupported board, or any read error all + // degrade to null — a missing sensor never fails the report). Same nullable contract as + // the per-disk SmartSummary.TemperatureC. Sourced from sysfs (hwmon / thermal zones). + // Cross-repo wire field (slice 9) — the hub's HostMetrics copy + golden carry it too. + CPUTempC *int `json:"cpu_temp_c"` } // Guest is one LXC. The agent reports vmid; the hub derives the guest PK diff --git a/internal/hub/report_test.go b/internal/hub/report_test.go index 11a242c..313c65e 100644 --- a/internal/hub/report_test.go +++ b/internal/hub/report_test.go @@ -16,6 +16,7 @@ func TestHostReport_FieldNamesAndEmptyCollections(t *testing.T) { MemoryTotalBytes: 16777216000, MemoryUsedBytes: 4194304000, MemoryPercent: 25.0, DiskTotalBytes: 152000000000, DiskUsedBytes: 30000000000, DiskPercent: 19.7, LoadAvg: []string{"0.10", "0.20", "0.15"}, UptimeSeconds: 86400, + CPUTempC: intp(47), // nullable scalar — set here so the "no null" invariant stays meaningful }, Guests: []Guest{{ VMID: 100, Name: "felhom-cust-acme", Status: "running", ControllerVersion: "", @@ -37,7 +38,7 @@ func TestHostReport_FieldNamesAndEmptyCollections(t *testing.T) { for _, field := range []string{ `"host_id":"demo-host-01"`, `"reported_at":`, `"agent_version":"0.3.0"`, `"cpu_percent":3.2`, `"memory_total_bytes":16777216000`, `"loadavg":["0.10","0.20","0.15"]`, - `"disk_percent":19.7`, `"uptime_seconds":86400`, + `"disk_percent":19.7`, `"uptime_seconds":86400`, `"cpu_temp_c":47`, `"vmid":100`, `"controller_version":""`, `"memory_bytes":2147483648`, `"cloudflared":{"status":"active"}`, // empty collections must be [] not null diff --git a/internal/hub/testdata/host-report.golden.json b/internal/hub/testdata/host-report.golden.json index 695f894..7309266 100644 --- a/internal/hub/testdata/host-report.golden.json +++ b/internal/hub/testdata/host-report.golden.json @@ -12,7 +12,8 @@ "disk_used_bytes": 30000000000, "disk_percent": 19.7, "loadavg": ["0.10", "0.20", "0.15"], - "uptime_seconds": 86400 + "uptime_seconds": 86400, + "cpu_temp_c": 47 }, "guests": [ { diff --git a/internal/localapi/host_metrics.go b/internal/localapi/host_metrics.go new file mode 100644 index 0000000..1fe30db --- /dev/null +++ b/internal/localapi/host_metrics.go @@ -0,0 +1,49 @@ +package localapi + +import ( + "net/http" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// Host metrics (slice 9, doc 03 §6). The de-privileged controller (slice 8C) can only see its own +// cgroup, so it cannot read host health itself. This endpoint re-serves the slice-4 collector's +// host + per-storage view to the customer so the controller can render the box's health. +// +// Host-wide, token-authed, fresh: the metrics are about the BOX (not per-guest), so any valid +// per-guest token gets the host-wide view (assumption: one customer per host — the home-server +// model). It is a live collect (fresh cpu%/temp), not the 15-min hub-report snapshot. + +// HostMetricsResponse is GET /host/metrics: the host block + per-storage capacity targets. +type HostMetricsResponse struct { + VMID int `json:"vmid"` + Host hub.HostMetrics `json:"host"` // cpu%/mem/load/uptime/cpu_temp_c + StorageTargets []hub.StorageTarget `json:"storage_targets"` // per-storage total/used/thin-pool/SMART temp+wear +} + +// handleHostMetrics serves a fresh host-health snapshot. The host block comes from a live +// collector read; the per-storage capacity comes from the observer (the same source the hub +// report uses). Best-effort on storage: a storage-view error still returns the host block (the +// CPU/mem/temp view is the headline) with an empty targets list. +func (s *Server) handleHostMetrics(w http.ResponseWriter, r *http.Request, vmid int) { + if s.hostMetrics == nil { + writeErr(w, http.StatusServiceUnavailable, "host metrics not configured on this host") + return + } + host, err := s.hostMetrics.HostMetricsNow(r.Context()) + if err != nil { + s.logger.Error("local-api: /host/metrics collect", "vmid", vmid, "err", err) + writeErr(w, http.StatusBadGateway, "could not read host metrics") + return + } + targets, err := s.storage.Observe(r.Context()) + if err != nil { + // Host health is the headline — don't sink it on a storage-view hiccup. + s.logger.Warn("local-api: /host/metrics storage view unavailable", "vmid", vmid, "err", err) + targets = []hub.StorageTarget{} + } + if targets == nil { + targets = []hub.StorageTarget{} + } + writeOK(w, HostMetricsResponse{VMID: vmid, Host: host, StorageTargets: targets}) +} diff --git a/internal/localapi/host_metrics_test.go b/internal/localapi/host_metrics_test.go new file mode 100644 index 0000000..786c726 --- /dev/null +++ b/internal/localapi/host_metrics_test.go @@ -0,0 +1,144 @@ +package localapi + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// fakeHostMetrics is a HostMetricsProvider returning a fixed host block (or an error). +type fakeHostMetrics struct { + host hub.HostMetrics + err error +} + +func (f fakeHostMetrics) HostMetricsNow(context.Context) (hub.HostMetrics, error) { + return f.host, f.err +} + +func newHostMetricsServer(t *testing.T, hm HostMetricsProvider, sv StorageView) http.Handler { + t.Helper() + if sv == nil { + sv = fakeStorage{} + } + srv, err := NewServer(Options{ + ListenAddr: "127.0.0.1:0", + Guests: &fakeGuests{}, + Backups: &fakeBackups{}, + Store: &fakeStore{}, + Storage: sv, + Tokens: staticTokens{"A": 8200, "B": 9300}, + HostMetrics: hm, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("new server: %v", err) + } + srv.baseCtx = context.Background() + return srv.Handler() +} + +func cpuTempPtr(v int) *int { return &v } + +// A valid token gets a populated host + storage view (host-wide, token-authed). +func TestHostMetrics_PopulatedWithValidToken(t *testing.T) { + hm := fakeHostMetrics{host: hub.HostMetrics{ + Node: "demo-felhom", CPUPercent: 12.5, MemoryTotalBytes: 16 << 30, MemoryUsedBytes: 4 << 30, + MemoryPercent: 25, UptimeSeconds: 86400, LoadAvg: []string{"0.10", "0.20", "0.15"}, + CPUTempC: cpuTempPtr(46), + }} + sv := fakeStorage{targets: []hub.StorageTarget{ + {Name: "local", Type: hub.StorageTypeLocal, State: hub.StorageStateAttached, Reachable: true, + TotalBytes: 100 << 30, UsedBytes: 20 << 30, UsedFraction: 0.2}, + }} + h := newHostMetricsServer(t, hm, sv) + + w := do(t, h, "GET", "/host/metrics", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("GET /host/metrics: got %d, want 200 (body=%s)", w.Code, w.Body.String()) + } + var env struct { + OK bool `json:"ok"` + Data HostMetricsResponse `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil { + t.Fatalf("decode: %v", err) + } + if !env.OK { + t.Fatal("ok=false") + } + if env.Data.VMID != 8200 { + t.Errorf("vmid = %d, want 8200 (token's guest)", env.Data.VMID) + } + if env.Data.Host.Node != "demo-felhom" || env.Data.Host.CPUPercent != 12.5 { + t.Errorf("host = %+v", env.Data.Host) + } + if env.Data.Host.CPUTempC == nil || *env.Data.Host.CPUTempC != 46 { + t.Errorf("cpu_temp_c = %v, want 46", env.Data.Host.CPUTempC) + } + if len(env.Data.StorageTargets) != 1 || env.Data.StorageTargets[0].Name != "local" { + t.Errorf("storage targets = %+v", env.Data.StorageTargets) + } +} + +// Null cpu_temp_c marshals as JSON null (the controller renders "n/a"). +func TestHostMetrics_NullTempSerializes(t *testing.T) { + hm := fakeHostMetrics{host: hub.HostMetrics{Node: "n", LoadAvg: []string{}, CPUTempC: nil}} + h := newHostMetricsServer(t, hm, nil) + w := do(t, h, "GET", "/host/metrics", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("got %d, want 200", w.Code) + } + // The raw JSON must contain `"cpu_temp_c":null` (a stable key, not omitted). + if got := w.Body.String(); !strings.Contains(got, `"cpu_temp_c":null`) { + t.Errorf("body missing cpu_temp_c null: %s", got) + } +} + +// No token → 401, and the collector is never invoked. +func TestHostMetrics_Requires401WithoutToken(t *testing.T) { + called := false + hm := callbackHostMetrics{fn: func() { called = true }} + h := newHostMetricsServer(t, hm, nil) + if w := do(t, h, "GET", "/host/metrics", "", ""); w.Code != http.StatusUnauthorized { + t.Fatalf("absent token: got %d, want 401", w.Code) + } + if w := do(t, h, "GET", "/host/metrics", "bogus", ""); w.Code != http.StatusUnauthorized { + t.Fatalf("unknown token: got %d, want 401", w.Code) + } + if called { + t.Fatal("host metrics collected despite failed auth") + } +} + +// When no provider is wired the endpoint reports "not configured" (503), not a crash. +func TestHostMetrics_NotConfigured(t *testing.T) { + h := newHostMetricsServer(t, nil, nil) + if w := do(t, h, "GET", "/host/metrics", "A", ""); w.Code != http.StatusServiceUnavailable { + t.Fatalf("got %d, want 503 (not configured)", w.Code) + } +} + +// A cross-guest probe (?vmid=other) is refused 403 even though the data is host-wide — the +// self-scoping invariant is uniform across endpoints. +func TestHostMetrics_CrossGuestQueryRefused(t *testing.T) { + hm := fakeHostMetrics{host: hub.HostMetrics{Node: "n", LoadAvg: []string{}}} + h := newHostMetricsServer(t, hm, nil) + if w := do(t, h, "GET", "/host/metrics?vmid=9300", "A", ""); w.Code != http.StatusForbidden { + t.Fatalf("cross-guest query: got %d, want 403", w.Code) + } +} + +// callbackHostMetrics records that the collector was invoked (to assert it is NOT on a 401). +type callbackHostMetrics struct{ fn func() } + +func (c callbackHostMetrics) HostMetricsNow(context.Context) (hub.HostMetrics, error) { + c.fn() + return hub.HostMetrics{LoadAvg: []string{}}, nil +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index 02742cf..50d08d1 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -54,6 +54,13 @@ type TokenAuthority interface { Lookup(token string) (int, bool) } +// HostMetricsProvider does a FRESH host-metrics collect (cpu%/mem/load/uptime/cpu-temp) for +// GET /host/metrics (slice 9). Satisfied by *hub.Collector (which reuses the slice-4 collector — +// no duplicate collection). Optional: when nil, /host/metrics reports "not configured". +type HostMetricsProvider interface { + HostMetricsNow(ctx context.Context) (hub.HostMetrics, error) +} + // Options configures a Server. type Options struct { ListenAddr string // bridge IP:port @@ -73,7 +80,11 @@ type Options struct { Disks DiskOps DiskGate StorageGate Guests2 GuestLister - Logger *slog.Logger + // HostMetrics serves GET /host/metrics (slice 9) — host-wide health (cpu%/mem/load/uptime/ + // cpu-temp) + per-storage capacity, host-wide and token-authed (one-customer-per-host). When + // nil the endpoint reports "not configured" (host still reports/reconciles). + HostMetrics HostMetricsProvider + Logger *slog.Logger } // defaultBackupCadence is the fallback /backup/due window when none is configured. @@ -117,6 +128,8 @@ type Server struct { diskGate StorageGate // slice 8C (optional) guestList GuestLister // slice 8C (optional) + hostMetrics HostMetricsProvider // slice 9 (optional) + jobsMu sync.Mutex jobs map[int]*backupJob // per-guest backup job state (slice 8B) @@ -149,10 +162,11 @@ func NewServer(o Options) (*Server, error) { cadence: cadence, logger: o.Logger, now: func() time.Time { return time.Now().UTC() }, - disks: o.Disks, - diskGate: o.DiskGate, - guestList: o.Guests2, - jobs: map[int]*backupJob{}, + disks: o.Disks, + diskGate: o.DiskGate, + guestList: o.Guests2, + hostMetrics: o.HostMetrics, + jobs: map[int]*backupJob{}, }, nil } @@ -166,6 +180,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /backup/due", s.withGuest(s.handleBackupDue)) mux.HandleFunc("GET /backup/status", s.withGuest(s.handleBackupStatus)) mux.HandleFunc("GET /restore-test/status", s.withGuest(s.handleRestoreTestStatus)) + // Host metrics (slice 9): host-wide health + per-storage capacity for the customer's monitoring + // view. Host-wide, token-authed, fresh (a live collect, not the 15-min hub snapshot). + mux.HandleFunc("GET /host/metrics", s.withGuest(s.handleHostMetrics)) // Disk management (slice 8C) — self-scoped; format routes through the data-bearing classifier+gate. mux.HandleFunc("GET /disks", s.withGuest(s.handleDisks)) mux.HandleFunc("POST /disks/assign", s.withGuest(s.handleDiskAssign))