v0.5.0-rc1: slice 5 Phase A — storage observe/report + watchdog (read-only, live)
Fill the slice-3 storage_targets stub and add the fast-poll storage watchdog. Read-only this phase; the host-root surface (mounts/SMART/grow/destructive gate) is Phase B. Hub-owned desired manifest is slice 10, so reconcile against it is built-but-unfed. - internal/storage: StorageTarget wire contract, durable_id derivation per type, HostReader seam (procfs/sysfs, root-free), Observer (storage_targets from ListStorage/NodeStorage + host reads, lvmthin thin-pool fill), and the watchdog (third daemon goroutine; debounced out-of-band report on a known target's attach/disconnect transition). - proxmox.Storage: additive parse-only config fields (durable_id sources). - collector StorageObserver seam; Loop.SetTrigger out-of-band report; daemon runs the watchdog as a third goroutine; StorageConfig knobs. - cross-repo golden kept byte-identical with felhom.eu/hub; bidirectional key-set test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,398 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// thinPoolWarnFraction is the lvmthin DATA-fill level above which the observer logs a
|
||||
// prominent warning. A full thin-pool corrupts EVERY guest on it (the storage analog of a
|
||||
// single-node OOM), so it must be visible early — well before slice-10 policy exists.
|
||||
const thinPoolWarnFraction = 0.85
|
||||
|
||||
// StorageAPI is the read-only Proxmox surface the observer needs. *proxmox.Client
|
||||
// satisfies it. ListStorage (cluster) carries the type-specific config (server/export/
|
||||
// vgname/thinpool/fingerprint) that NodeStorage may omit; NodeStorage carries live usage
|
||||
// + the per-node active flag. The observer joins them by storage name.
|
||||
type StorageAPI interface {
|
||||
Node() string
|
||||
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||
NodeStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||
}
|
||||
|
||||
// Observer builds the observed storage view from Proxmox + non-privileged host reads.
|
||||
type Observer struct {
|
||||
api StorageAPI
|
||||
host HostReader
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewObserver builds an Observer. host defaults to a ProcHostReader; logger to the
|
||||
// default. A nil api makes Observe/Known return an error (misconfiguration), never panic.
|
||||
func NewObserver(api StorageAPI, host HostReader, logger *slog.Logger) *Observer {
|
||||
if host == nil {
|
||||
host = NewProcHostReader()
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Observer{api: api, host: host, logger: logger}
|
||||
}
|
||||
|
||||
// observed is the rich internal view of one target, from which both the reported
|
||||
// hub.StorageTarget and the watchdog's KnownTarget are projected.
|
||||
type observed struct {
|
||||
target hub.StorageTarget
|
||||
known KnownTarget
|
||||
}
|
||||
|
||||
// Observe builds the reported []hub.StorageTarget. A non-nil error means the Proxmox read
|
||||
// failed (the collector then omits storage from this cycle's report but still sends the
|
||||
// rest). The returned slice is always non-nil so it marshals as [] when empty.
|
||||
func (o *Observer) Observe(ctx context.Context) ([]hub.StorageTarget, error) {
|
||||
snap, err := o.snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]hub.StorageTarget, 0, len(snap))
|
||||
for _, s := range snap {
|
||||
out = append(out, s.target)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Known projects the snapshot to the watchdog's lightweight KnownTarget set. Same Proxmox
|
||||
// + host reads as Observe — callers that poll it fast should wrap it in a cache (the
|
||||
// watchdog uses CachingKnownTargets).
|
||||
func (o *Observer) Known(ctx context.Context) ([]KnownTarget, error) {
|
||||
snap, err := o.snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]KnownTarget, 0, len(snap))
|
||||
for _, s := range snap {
|
||||
out = append(out, s.known)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// snapshot does the full build: join cluster config + node usage, then derive each
|
||||
// target's identity, state, class hint, and (for lvmthin) thin-pool fill from host reads.
|
||||
func (o *Observer) snapshot(ctx context.Context) ([]observed, error) {
|
||||
if o.api == nil {
|
||||
return nil, fmt.Errorf("storage: no proxmox api configured")
|
||||
}
|
||||
cluster, err := o.api.ListStorage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: ListStorage: %w", err)
|
||||
}
|
||||
cfgByName := make(map[string]proxmox.Storage, len(cluster))
|
||||
for _, c := range cluster {
|
||||
cfgByName[c.Storage] = c
|
||||
}
|
||||
nodeStores, err := o.api.NodeStorage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: NodeStorage: %w", err)
|
||||
}
|
||||
mounts, err := o.host.Mounts()
|
||||
if err != nil {
|
||||
// Host mount read failed: degrade rather than fail the whole report — Proxmox
|
||||
// usage/active is still meaningful; we just lose mount-derived fields.
|
||||
o.logger.Warn("storage: reading mounts failed; mount/device fields degraded", "err", err)
|
||||
mounts = nil
|
||||
}
|
||||
|
||||
out := make([]observed, 0, len(nodeStores))
|
||||
for _, ns := range nodeStores {
|
||||
// Overlay the cluster config (server/export/vgname/...) onto the node entry.
|
||||
s := mergeConfig(ns, cfgByName[ns.Storage])
|
||||
out = append(out, o.build(s, mounts))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// build derives one observed target from a merged Storage entry + the mount table.
|
||||
func (o *Observer) build(s proxmox.Storage, mounts []Mount) observed {
|
||||
category := categorize(s.Type)
|
||||
|
||||
// Resolve the backing device + mount path for dir-like targets.
|
||||
var backingDevice, mountPath string
|
||||
var exactMount bool
|
||||
if category == catDir {
|
||||
if dev, mp, ok := exactMountDevice(mounts, s.Path); ok {
|
||||
backingDevice, mountPath, exactMount = dev, mp, true
|
||||
} else if dev, ok := containingMountDevice(mounts, s.Path); ok {
|
||||
backingDevice = dev // for the class hint only; not its own mount
|
||||
}
|
||||
}
|
||||
|
||||
// Type: distinguish builtin local / removable USB / fixed local-dir within "dir".
|
||||
removable, removableKnown := false, false
|
||||
if category == catDir && backingDevice != "" {
|
||||
removable, removableKnown = o.host.Removable(backingDevice)
|
||||
}
|
||||
typ := reportType(s, category, removable, removableKnown)
|
||||
|
||||
// durable_id (DR-load-bearing).
|
||||
var uuid string
|
||||
if category == catDir && backingDevice != "" {
|
||||
uuid, _ = o.host.ResolveUUID(backingDevice)
|
||||
}
|
||||
durableID := deriveDurableID(typ, s, backingDevice, uuid)
|
||||
|
||||
// Reachability + state.
|
||||
reachable := o.reachable(typ, category, s, backingDevice, exactMount)
|
||||
state := hub.StorageStateAttached
|
||||
if !reachable {
|
||||
state = hub.StorageStateDisconnected
|
||||
}
|
||||
|
||||
// Class hint (rotational; local block-backed only — a HINT, never authoritative).
|
||||
classHint := ""
|
||||
if category == catDir && backingDevice != "" {
|
||||
if rot, ok := o.host.Rotational(backingDevice); ok {
|
||||
if rot {
|
||||
classHint = "slow"
|
||||
} else {
|
||||
classHint = "fast"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tgt := hub.StorageTarget{
|
||||
Name: s.Storage,
|
||||
Type: typ,
|
||||
DurableID: durableID,
|
||||
State: state,
|
||||
Reachable: reachable,
|
||||
TotalBytes: s.Total,
|
||||
UsedBytes: s.Used,
|
||||
AvailBytes: s.Avail,
|
||||
UsedFraction: usedFraction(s),
|
||||
Content: s.Content,
|
||||
MountPath: mountPath,
|
||||
BackingDevice: backingDevice,
|
||||
ClassHint: classHint,
|
||||
Role: "", // hub-owned; not derivable from a Proxmox def (slice 10)
|
||||
Smart: hub.SmartSummary{Health: hub.SmartUnknown},
|
||||
}
|
||||
|
||||
// Thin-pool DATA fill: surfaced prominently for lvmthin (metadata fill is Phase B/lvs).
|
||||
if typ == hub.StorageTypeLVMThin {
|
||||
frac := usedFraction(s)
|
||||
tgt.ThinPool = &hub.ThinPoolFill{DataUsedFraction: frac}
|
||||
if frac >= thinPoolWarnFraction {
|
||||
o.logger.Warn("storage: lvmthin pool data fill is high (a full pool corrupts every guest on it)",
|
||||
"storage", s.Storage, "data_used_fraction", frac)
|
||||
}
|
||||
}
|
||||
|
||||
return observed{
|
||||
target: tgt,
|
||||
known: KnownTarget{
|
||||
Name: s.Storage,
|
||||
Type: typ,
|
||||
DurableID: durableID,
|
||||
Network: category == catNetwork,
|
||||
MountBacked: typ == hub.StorageTypeUSB || typ == hub.StorageTypeLocalDir,
|
||||
BackingDevice: backingDevice,
|
||||
MountPath: s.Path,
|
||||
ReachEndpoint: reachEndpoint(typ, s),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// reachable decides whether the target is currently usable.
|
||||
// - usb / local-dir: a Felhom extra/removable dir storage is realized as its OWN
|
||||
// mountpoint, so reachable = it is currently an exact mount AND its device node exists.
|
||||
// Not-its-own-mount = unplugged/unmounted = disconnected. This is the fast USB-drop
|
||||
// signal — we deliberately do NOT fall through to PVE's active flag, because the
|
||||
// mountpoint directory still exists on the root fs when the device is gone, so active
|
||||
// can read stale-attached.
|
||||
// - local (builtin PVE "local"): lives within the root fs by design, so trust active.
|
||||
// - network (nfs/cifs/pbs) and block-pool (lvmthin/lvm): trust PVE's active flag — PVE
|
||||
// actively probes these and flips active=0 when down.
|
||||
func (o *Observer) reachable(typ string, category storageCategory, s proxmox.Storage, backingDevice string, exactMount bool) bool {
|
||||
switch typ {
|
||||
case hub.StorageTypeUSB, hub.StorageTypeLocalDir:
|
||||
return exactMount && (backingDevice == "" || o.host.DeviceExists(backingDevice))
|
||||
default:
|
||||
// local, lvmthin, lvm, nfs, cifs, pbs.
|
||||
_ = category
|
||||
return s.Active == 1
|
||||
}
|
||||
}
|
||||
|
||||
// usedFraction prefers Proxmox's reported used_fraction, falling back to used/total.
|
||||
func usedFraction(s proxmox.Storage) float64 {
|
||||
if s.UsedFraction > 0 {
|
||||
return s.UsedFraction
|
||||
}
|
||||
if s.Total > 0 {
|
||||
return float64(s.Used) / float64(s.Total)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// storageCategory groups Proxmox storage types by how state/identity are derived.
|
||||
type storageCategory int
|
||||
|
||||
const (
|
||||
catDir storageCategory = iota // dir-backed (local/usb/local-dir)
|
||||
catNetwork // nfs/cifs/pbs
|
||||
catBlock // lvmthin/lvm
|
||||
catOther
|
||||
)
|
||||
|
||||
func categorize(pxType string) storageCategory {
|
||||
switch pxType {
|
||||
case "dir":
|
||||
return catDir
|
||||
case "nfs", "cifs", "smb", "pbs":
|
||||
return catNetwork
|
||||
case "lvmthin", "lvm":
|
||||
return catBlock
|
||||
default:
|
||||
return catOther
|
||||
}
|
||||
}
|
||||
|
||||
// reportType maps a Proxmox storage type to the reported vocabulary, splitting "dir" into
|
||||
// builtin local / removable usb / fixed local-dir.
|
||||
func reportType(s proxmox.Storage, category storageCategory, removable, removableKnown bool) string {
|
||||
switch category {
|
||||
case catDir:
|
||||
if s.Storage == "local" {
|
||||
return hub.StorageTypeLocal
|
||||
}
|
||||
if removableKnown && removable {
|
||||
return hub.StorageTypeUSB
|
||||
}
|
||||
return hub.StorageTypeLocalDir
|
||||
case catNetwork:
|
||||
switch s.Type {
|
||||
case "nfs":
|
||||
return hub.StorageTypeNFS
|
||||
case "cifs", "smb":
|
||||
return hub.StorageTypeCIFS
|
||||
case "pbs":
|
||||
return hub.StorageTypePBS
|
||||
}
|
||||
case catBlock:
|
||||
if s.Type == "lvmthin" {
|
||||
return hub.StorageTypeLVMThin
|
||||
}
|
||||
return s.Type // "lvm" (thick) passes through
|
||||
}
|
||||
return s.Type
|
||||
}
|
||||
|
||||
// reachEndpoint builds the host:port the watchdog dials for a network target's
|
||||
// reachability check (default ports per protocol). "" for non-network targets.
|
||||
func reachEndpoint(typ string, s proxmox.Storage) string {
|
||||
if s.Server == "" {
|
||||
return ""
|
||||
}
|
||||
switch typ {
|
||||
case hub.StorageTypeNFS:
|
||||
return netJoin(s.Server, "2049")
|
||||
case hub.StorageTypeCIFS:
|
||||
return netJoin(s.Server, "445")
|
||||
case hub.StorageTypePBS:
|
||||
return netJoin(s.Server, "8007")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func netJoin(host, port string) string {
|
||||
if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") {
|
||||
host = "[" + host + "]" // IPv6 literal
|
||||
}
|
||||
return host + ":" + port
|
||||
}
|
||||
|
||||
// exactMountDevice finds the mount whose mountpoint EXACTLY equals path (the target is its
|
||||
// own mount — the meaningful state for a USB/extra disk).
|
||||
func exactMountDevice(mounts []Mount, path string) (device, mountPoint string, ok bool) {
|
||||
if path == "" {
|
||||
return "", "", false
|
||||
}
|
||||
clean := cleanMountPath(path)
|
||||
for _, m := range mounts {
|
||||
if cleanMountPath(m.MountPoint) == clean {
|
||||
return m.Device, m.MountPoint, true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// containingMountDevice finds the device of the longest mountpoint that is a prefix of
|
||||
// path (the filesystem that path lives on) — used only for the class-hint disk lookup.
|
||||
func containingMountDevice(mounts []Mount, path string) (device string, ok bool) {
|
||||
if path == "" {
|
||||
return "", false
|
||||
}
|
||||
clean := cleanMountPath(path)
|
||||
best := -1
|
||||
for _, m := range mounts {
|
||||
mp := cleanMountPath(m.MountPoint)
|
||||
if clean == mp || strings.HasPrefix(clean, mp+"/") || mp == "/" {
|
||||
if len(mp) > best {
|
||||
best, device, ok = len(mp), m.Device, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return device, ok
|
||||
}
|
||||
|
||||
func cleanMountPath(p string) string {
|
||||
p = strings.TrimRight(p, "/")
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// mergeConfig overlays the cluster-def config fields (which the per-node entry may omit)
|
||||
// onto a node-storage entry, keeping the node's live usage/active values.
|
||||
func mergeConfig(node, cluster proxmox.Storage) proxmox.Storage {
|
||||
if cluster.Storage == "" {
|
||||
return node
|
||||
}
|
||||
if node.Type == "" {
|
||||
node.Type = cluster.Type
|
||||
}
|
||||
if node.Server == "" {
|
||||
node.Server = cluster.Server
|
||||
}
|
||||
if node.Export == "" {
|
||||
node.Export = cluster.Export
|
||||
}
|
||||
if node.Share == "" {
|
||||
node.Share = cluster.Share
|
||||
}
|
||||
if node.Datastore == "" {
|
||||
node.Datastore = cluster.Datastore
|
||||
}
|
||||
if node.Fingerprint == "" {
|
||||
node.Fingerprint = cluster.Fingerprint
|
||||
}
|
||||
if node.VGName == "" {
|
||||
node.VGName = cluster.VGName
|
||||
}
|
||||
if node.ThinPool == "" {
|
||||
node.ThinPool = cluster.ThinPool
|
||||
}
|
||||
if node.Path == "" {
|
||||
node.Path = cluster.Path
|
||||
}
|
||||
if node.Content == "" {
|
||||
node.Content = cluster.Content
|
||||
}
|
||||
return node
|
||||
}
|
||||
Reference in New Issue
Block a user