v0.5.0-rc1: slice 5 Phase A — storage observe/report + watchdog (read-only, live)

Fill the slice-3 storage_targets stub and add the fast-poll storage watchdog.
Read-only this phase; the host-root surface (mounts/SMART/grow/destructive gate)
is Phase B. Hub-owned desired manifest is slice 10, so reconcile against it is
built-but-unfed.

- internal/storage: StorageTarget wire contract, durable_id derivation per type,
  HostReader seam (procfs/sysfs, root-free), Observer (storage_targets from
  ListStorage/NodeStorage + host reads, lvmthin thin-pool fill), and the watchdog
  (third daemon goroutine; debounced out-of-band report on a known target's
  attach/disconnect transition).
- proxmox.Storage: additive parse-only config fields (durable_id sources).
- collector StorageObserver seam; Loop.SetTrigger out-of-band report; daemon runs
  the watchdog as a third goroutine; StorageConfig knobs.
- cross-repo golden kept byte-identical with felhom.eu/hub; bidirectional key-set test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 09:59:05 +02:00
parent 1af21a6cac
commit 27b68f043b
22 changed files with 2129 additions and 103 deletions
+34 -4
View File
@@ -22,11 +22,21 @@ type proxmoxReader interface {
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
}
// StorageObserver is the seam the storage layer (internal/storage) plugs into to fill the
// report's storage_targets. Defined here (consumer-side) so hub does NOT import storage —
// storage imports hub for the wire type, and main.go wires the concrete observer in. Same
// pattern as proxmoxReader / CloudflaredProber. A nil observer (slice-3 behaviour, or a
// host with no storage layer) yields an empty []StorageTarget without error.
type StorageObserver interface {
Observe(ctx context.Context) ([]StorageTarget, error)
}
// Collector builds a HostReport from read-only sources. All deps are behind narrow
// interfaces for unit testing.
type Collector struct {
px proxmoxReader
cf CloudflaredProber
storage StorageObserver
hostID string
agentVersion string
logger *slog.Logger
@@ -34,14 +44,15 @@ type Collector struct {
}
// NewCollector builds a collector. hostID echoes config.Hub.HostID; agentVersion is
// the binary version.
func NewCollector(px proxmoxReader, cf CloudflaredProber, hostID, agentVersion string, logger *slog.Logger) *Collector {
// the binary version. storage may be nil (storage_targets emitted empty).
func NewCollector(px proxmoxReader, cf CloudflaredProber, storage StorageObserver, hostID, agentVersion string, logger *slog.Logger) *Collector {
if logger == nil {
logger = slog.Default()
}
return &Collector{
px: px,
cf: cf,
storage: storage,
hostID: hostID,
agentVersion: agentVersion,
logger: logger,
@@ -65,8 +76,9 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
AgentVersion: c.agentVersion,
Host: hostMetrics(c.px.Node(), ns),
Guests: c.collectGuests(ctx),
// Defined-but-empty this slice (slices 5/6). Non-nil so they marshal as [].
StorageTargets: []StorageTarget{},
// storage_targets populated this slice (slice 5) via the observer; the rest stay
// defined-but-empty (slice 6). Non-nil so they marshal as [].
StorageTargets: c.collectStorage(ctx),
Backups: []Backup{},
RestoreTests: []RestoreTest{},
PBSSnapshots: []PBSSnapshot{},
@@ -129,6 +141,24 @@ func (c *Collector) collectGuests(ctx context.Context) []Guest {
return guests
}
// collectStorage builds the storage_targets via the observer. A nil observer (no storage
// layer wired) or an observe error degrades to an empty list — storage detail is
// best-effort and must never sink the heartbeat (host liveness is the priority).
func (c *Collector) collectStorage(ctx context.Context) []StorageTarget {
if c.storage == nil {
return []StorageTarget{}
}
targets, err := c.storage.Observe(ctx)
if err != nil {
c.logger.Warn("hub: storage observe failed; reporting no storage targets", "err", err)
return []StorageTarget{}
}
if targets == nil {
return []StorageTarget{}
}
return targets
}
func (c *Collector) cloudflaredStatus(ctx context.Context) string {
if c.cf == nil {
return "unknown"
+39 -4
View File
@@ -20,6 +20,41 @@ func newTestNodeStatus() proxmox.NodeStatus {
return ns
}
// fakeObserver is a StorageObserver returning fixed targets (or an error).
type fakeObserver struct {
targets []StorageTarget
err error
}
func (f fakeObserver) Observe(context.Context) ([]StorageTarget, error) { return f.targets, f.err }
func TestCollect_StorageTargetsFromObserver(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
obs := fakeObserver{targets: []StorageTarget{
{Name: "local-lvm", Type: StorageTypeLVMThin, State: StorageStateAttached, Reachable: true},
}}
c := NewCollector(px, fakeProber{status: "active"}, obs, "h", "0.5.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
}
if len(r.StorageTargets) != 1 || r.StorageTargets[0].Name != "local-lvm" {
t.Fatalf("storage targets = %+v", r.StorageTargets)
}
}
func TestCollect_StorageObserverErrorDegradesToEmpty(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, "h", "0.5.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("a storage observe error must not sink the heartbeat: %v", err)
}
if r.StorageTargets == nil || len(r.StorageTargets) != 0 {
t.Errorf("storage targets must degrade to empty non-nil, got %+v", r.StorageTargets)
}
}
func TestCollect_HostAndGuests(t *testing.T) {
px := &fakePx{
node: "demo-felhom",
@@ -29,7 +64,7 @@ func TestCollect_HostAndGuests(t *testing.T) {
},
cfg: map[int]proxmox.GuestConfig{100: {Cores: 2, Memory: 2048}},
}
c := NewCollector(px, fakeProber{status: "active"}, "demo-host-01", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{status: "active"}, nil, "demo-host-01", "0.3.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
@@ -69,7 +104,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {
cfg: map[int]proxmox.GuestConfig{100: {Cores: 2}},
cfgErr: map[int]error{200: errors.New("config read failed")},
}
c := NewCollector(px, fakeProber{status: "active"}, "h", "0.3.1", quietLogger())
c := NewCollector(px, fakeProber{status: "active"}, nil, "h", "0.3.1", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("a per-guest failure must NOT fail the whole report: %v", err)
@@ -90,7 +125,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {
func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {
px := &fakePx{node: "n", nsErr: errors.New("proxmox down")}
c := NewCollector(px, fakeProber{status: "active"}, "h", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{status: "active"}, nil, "h", "0.3.0", quietLogger())
if _, err := c.Collect(context.Background()); err == nil {
t.Fatal("NodeStatus failure must be a hard error (no useful report)")
}
@@ -98,7 +133,7 @@ func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {
func TestCollect_CloudflaredProbeErrorIsUnknown(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, "h", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, "h", "0.3.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("cloudflared failure must not be fatal: %v", err)
+34 -3
View File
@@ -24,7 +24,8 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
t.Fatalf("golden is not valid JSON: %v", err)
}
// A constructed report mirroring the golden's populated shape (guests[0] has spec).
// A constructed report mirroring the golden's populated shape: guests[0] has spec,
// storage_targets[0] is an lvmthin (so its thin_pool + smart sub-objects are exercised).
report := &HostReport{
HostID: "demo-host-01", ReportedAt: "2026-06-08T12:00:00Z", AgentVersion: "0.3.1",
Host: HostMetrics{Node: "demo-felhom", LoadAvg: []string{"0.10"}},
@@ -32,9 +33,22 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
{VMID: 100, Name: "a", Status: "running", ControllerVersion: "", Spec: &GuestSpec{Cores: 2}},
{VMID: 101, Name: "b", Status: "stopped", ControllerVersion: ""},
},
StorageTargets: []StorageTarget{}, Backups: []Backup{}, RestoreTests: []RestoreTest{},
StorageTargets: []StorageTarget{
{
Name: "local-lvm", Type: StorageTypeLVMThin, DurableID: "pve/data",
State: StorageStateAttached, Reachable: true, ClassHint: "fast",
ThinPool: &ThinPoolFill{DataUsedFraction: 0.42},
Smart: SmartSummary{Health: SmartUnknown},
},
{
Name: "usb-backup", Type: StorageTypeUSB, DurableID: "uuid:x",
State: StorageStateAttached, Reachable: true,
Smart: SmartSummary{Health: SmartUnknown},
},
},
Backups: []Backup{}, RestoreTests: []RestoreTest{},
PBSSnapshots: []PBSSnapshot{}, AuditTail: []AuditEntry{},
Cloudflared: Cloudflared{Status: "active"},
Cloudflared: Cloudflared{Status: "active"},
}
b, _ := json.Marshal(report)
var got map[string]any
@@ -44,6 +58,23 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
assertSameKeys(t, "host", golden["host"], got["host"])
assertSameKeys(t, "guests[0]",
firstElem(golden["guests"]), firstElem(got["guests"]))
// storage_targets[0] is the slice-5 addition — assert its full key set and both
// sub-objects (smart always present; thin_pool present for the lvmthin element).
gst := firstElem(golden["storage_targets"])
sst := firstElem(got["storage_targets"])
assertSameKeys(t, "storage_targets[0]", gst, sst)
assertSameKeys(t, "storage_targets[0].smart", field(gst, "smart"), field(sst, "smart"))
assertSameKeys(t, "storage_targets[0].thin_pool", field(gst, "thin_pool"), field(sst, "thin_pool"))
}
// field extracts a nested object value from a decoded JSON map (nil if absent/not a map).
func field(v any, key string) any {
m, ok := v.(map[string]any)
if !ok {
return nil
}
return m[key]
}
func firstElem(v any) any {
+18
View File
@@ -30,6 +30,7 @@ type Loop struct {
client reporter
interval time.Duration
logger *slog.Logger
trigger <-chan struct{} // optional: an out-of-band report request (storage watchdog)
}
// NewLoop builds the loop. interval is the starting cadence (the hub may override it
@@ -41,6 +42,12 @@ func NewLoop(collector collectorIface, client reporter, interval time.Duration,
return &Loop{collector: collector, client: client, interval: interval, logger: logger}
}
// SetTrigger wires an out-of-band report channel. A receive on it runs one extra
// collect→report cycle immediately WITHOUT disturbing the regular ticker cadence — used by
// the storage watchdog to push a disconnect to the hub in seconds. The watchdog debounces,
// so this fires at most once per debounce window.
func (l *Loop) SetTrigger(ch <-chan struct{}) { l.trigger = ch }
// Run reports immediately, then on each tick, until ctx is cancelled (then nil).
func (l *Loop) Run(ctx context.Context) error {
interval := l.interval
@@ -60,6 +67,17 @@ func (l *Loop) Run(ctx context.Context) error {
interval = next
ticker.Reset(interval)
}
case <-l.trigger:
// Out-of-band report (storage watchdog). Run a cycle now; keep the regular
// cadence (do not reset the ticker). The envelope's interval is still adopted
// if it changed, mirroring the normal path.
l.logger.Info("hub: out-of-band report triggered (storage watchdog)")
next := l.cycle(ctx, interval)
if next != interval {
l.logger.Info("hub: poll interval changed", "from", interval, "to", next)
interval = next
ticker.Reset(interval)
}
}
}
}
+27
View File
@@ -122,6 +122,33 @@ func TestLoop_RunImmediateAndResilientAfterError(t *testing.T) {
}
}
func TestLoop_OutOfBandTriggerReportsImmediately(t *testing.T) {
// The storage watchdog's trigger channel runs an extra report between ticks (the slow
// cadence is 1h here, so any report within the window comes from the trigger).
var cn, rn int32
loop := NewLoop(
&fakeCollector{report: &HostReport{}, n: &cn},
&fakeReporter{env: &ControlEnvelope{}, n: &rn},
time.Hour, quietLogger())
trigger := make(chan struct{}, 1)
loop.SetTrigger(trigger)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- loop.Run(ctx) }()
// Immediate report fires first (1 collect). Then fire the trigger → one more report.
time.Sleep(20 * time.Millisecond)
trigger <- struct{}{}
time.Sleep(20 * time.Millisecond)
cancel()
<-done
if got := atomic.LoadInt32(&cn); got < 2 {
t.Errorf("collect calls = %d, want ≥2 (immediate + out-of-band trigger)", got)
}
}
func TestLoop_RunAdoptsSlowerInterval(t *testing.T) {
var cn, rn int32
loop := NewLoop(
+102 -3
View File
@@ -63,10 +63,109 @@ type Cloudflared struct {
}
// The following element types are declared now so the empty collections above are
// typed and slices 5/6 only fill them. No wire fields are committed yet.
// typed and slices 5/6 only fill them.
type StorageTarget struct{} // slice 5: storage manifest target fields TBD
type Backup struct{} // slice 6: per-target backup status fields TBD
// StorageTarget is one observed host storage target (doc 03 §7). It is the REPORTED
// shape — what the agent observes and tells the hub. The hub holds the AUTHORITATIVE
// manifest (desired class/role/policy/creds, slice 10); the agent reports what it sees
// and reconciles toward the hub's manifest. So `class_hint` is a rotational HINT, never
// authoritative class, and `role` is set only when derivable from an existing Proxmox
// storage definition (else empty — the hub owns it).
//
// This is a cross-repo contract DUPLICATED in felhom.eu/hub (no shared module yet);
// testdata/host-report.golden.json must stay byte-identical with the hub's copy and the
// bidirectional key-set test (contract_test.go) guards drift.
type StorageTarget struct {
Name string `json:"name"` // the Proxmox storage id (its name)
Type string `json:"type"` // local-dir | lvmthin | usb | nfs | cifs | pbs | local
DurableID string `json:"durable_id"` // fs-UUID (usb/local-dir) | server:export (nfs/cifs) | repo+fingerprint (pbs)
// State is the observed lifecycle state. attached (present & usable) | disconnected
// (a KNOWN target whose backing device/mount/reachability dropped) | decommissioned
// (hub-manifest state — built but not served until slice 10).
State string `json:"state"`
Reachable bool `json:"reachable"` // backing device present + mounted (local) / reachable (network)
TotalBytes int64 `json:"total_bytes"`
UsedBytes int64 `json:"used_bytes"`
AvailBytes int64 `json:"avail_bytes"`
UsedFraction float64 `json:"used_fraction"`
Content string `json:"content"` // Proxmox content list, e.g. "rootdir,images" / "backup,vztmpl"
MountPath string `json:"mount_path"` // host mountpoint (dir/usb); "" for network/lvm
BackingDevice string `json:"backing_device"` // resolved block device (e.g. /dev/sdb1); "" for network
// ClassHint is a fast|slow HINT derived from the backing disk's rotational flag — a
// hint only; the authoritative class is hub-owned (locked decision). "" when not
// derivable (network targets have no local rotational flag).
ClassHint string `json:"class_hint"`
// Role is primary|vzdump-target|pbs-offsite|bulk-data when derivable from the existing
// storage definition; else "" (the manifest role is hub-owned, slice 10).
Role string `json:"role"`
// ThinPool carries the lvmthin DATA fill prominently — a full thin-pool corrupts every
// guest on it (the storage analog of single-node OOM). Present ONLY for lvmthin targets;
// metadata fill is null until Phase B's privileged `lvs` read.
ThinPool *ThinPoolFill `json:"thin_pool,omitempty"`
// Smart is the disk-health summary, populated in Phase B via smartctl (sudoers). In
// Phase A it is {health:"UNKNOWN", <counters null>} — the keys are committed now so the
// contract is stable across both phases.
Smart SmartSummary `json:"smart"`
}
// ThinPoolFill is the lvmthin pool fill (doc 03 §7). DataUsedFraction is the live data
// fill (used/total of the lvmthin store); MetadataUsedFraction needs the privileged
// `lvs` read (Phase B) and is null until then.
type ThinPoolFill struct {
DataUsedFraction float64 `json:"data_used_fraction"`
MetadataUsedFraction *float64 `json:"metadata_used_fraction"`
}
// SmartSummary is a read-only disk-health summary. Health is PASSED|FAILING|UNKNOWN.
// Counters are pointers so "unknown / not-applicable for this device type" (e.g. a USB
// bridge that exposes no SMART, or NVMe counters on a SATA disk) is null, distinct from a
// real zero. The SATA set (reallocated/pending/offline-uncorrectable) and the NVMe set
// (critical_warning/media_errors/percentage_used) are both carried; a device populates
// only its own set. Filled in Phase B.
type SmartSummary struct {
Health string `json:"health"`
TemperatureC *int `json:"temperature_c"`
PowerOnHours *int `json:"power_on_hours"`
// SATA attributes.
ReallocatedSectors *int `json:"reallocated_sectors"`
PendingSectors *int `json:"pending_sectors"`
OfflineUncorrectable *int `json:"offline_uncorrectable"`
// NVMe attributes.
CriticalWarning *int `json:"critical_warning"`
MediaErrors *int `json:"media_errors"`
PercentageUsed *int `json:"percentage_used"`
}
// SMART health constants (the reported vocabulary).
const (
SmartPassed = "PASSED"
SmartFailing = "FAILING"
SmartUnknown = "UNKNOWN"
)
// Storage target type + state constants (the reported vocabulary; doc 03 §7).
const (
StorageTypeLocalDir = "local-dir"
StorageTypeLVMThin = "lvmthin"
StorageTypeUSB = "usb"
StorageTypeNFS = "nfs"
StorageTypeCIFS = "cifs"
StorageTypePBS = "pbs"
StorageTypeLocal = "local" // builtin dir storage (PVE "local")
StorageStateAttached = "attached"
StorageStateDisconnected = "disconnected"
StorageStateDecommissioned = "decommissioned"
)
type Backup struct{} // slice 6: per-target backup status fields TBD
type RestoreTest struct{} // slice 6: self-restore-test result fields TBD
type PBSSnapshot struct{} // slice 6: PBS snapshot inventory fields TBD
type AuditEntry struct{} // audit-log tail entry fields TBD
+57 -1
View File
@@ -29,7 +29,63 @@
"controller_version": ""
}
],
"storage_targets": [],
"storage_targets": [
{
"name": "local-lvm",
"type": "lvmthin",
"durable_id": "pve/data",
"state": "attached",
"reachable": true,
"total_bytes": 100000000000,
"used_bytes": 42000000000,
"avail_bytes": 58000000000,
"used_fraction": 0.42,
"content": "rootdir,images",
"mount_path": "",
"backing_device": "",
"class_hint": "fast",
"role": "",
"thin_pool": { "data_used_fraction": 0.42, "metadata_used_fraction": null },
"smart": {
"health": "UNKNOWN",
"temperature_c": null,
"power_on_hours": null,
"reallocated_sectors": null,
"pending_sectors": null,
"offline_uncorrectable": null,
"critical_warning": null,
"media_errors": null,
"percentage_used": null
}
},
{
"name": "usb-backup",
"type": "usb",
"durable_id": "uuid:0fc63daf-8483-4772-8e79-3d69d8477de4",
"state": "attached",
"reachable": true,
"total_bytes": 2000000000000,
"used_bytes": 500000000000,
"avail_bytes": 1500000000000,
"used_fraction": 0.25,
"content": "backup",
"mount_path": "/mnt/usb-backup",
"backing_device": "/dev/sdb1",
"class_hint": "slow",
"role": "",
"smart": {
"health": "UNKNOWN",
"temperature_c": null,
"power_on_hours": null,
"reallocated_sectors": null,
"pending_sectors": null,
"offline_uncorrectable": null,
"critical_warning": null,
"media_errors": null,
"percentage_used": null
}
}
],
"backups": [],
"restore_tests": [],
"pbs_snapshots": [],