package storage import ( "context" "fmt" "log/slog" "regexp" "strings" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // thinPoolWarnFraction is the lvmthin DATA-fill level above which the observer logs a // prominent warning. A full thin-pool corrupts EVERY guest on it (the storage analog of a // single-node OOM), so it must be visible early — well before slice-10 policy exists. const thinPoolWarnFraction = 0.85 // StorageAPI is the read-only Proxmox surface the observer needs. *proxmox.Client // satisfies it. ListStorage (cluster) carries the type-specific config (server/export/ // vgname/thinpool/fingerprint) that NodeStorage may omit; NodeStorage carries live usage // + the per-node active flag. The observer joins them by storage name. type StorageAPI interface { Node() string ListStorage(ctx context.Context) ([]proxmox.Storage, error) NodeStorage(ctx context.Context) ([]proxmox.Storage, error) } // Observer builds the observed storage view from Proxmox + non-privileged host reads. // In Phase B it also (optionally) enriches the reported view with the privileged reads — // SMART + thin-pool metadata — via HostOps; a nil ops keeps the Phase-A behaviour // (SMART UNKNOWN, metadata null). type Observer struct { api StorageAPI host HostReader ops HostOps logger *slog.Logger } // NewObserver builds an Observer. host defaults to a ProcHostReader; logger to the // default. ops is the privileged surface for SMART/lvs — nil disables those (Phase-A // behaviour). A nil api makes Observe/Known return an error (misconfiguration), never panic. func NewObserver(api StorageAPI, host HostReader, ops HostOps, logger *slog.Logger) *Observer { if host == nil { host = NewProcHostReader() } if logger == nil { logger = slog.Default() } return &Observer{api: api, host: host, ops: ops, logger: logger} } // observed is the rich internal view of one target, from which both the reported // hub.StorageTarget and the watchdog's KnownTarget are projected. src/cat are kept for // Observe-time privileged enrichment (NOT used by the watchdog's Known path). type observed struct { target hub.StorageTarget known KnownTarget src proxmox.Storage cat storageCategory } // Observe builds the reported []hub.StorageTarget. A non-nil error means the Proxmox read // failed (the collector then omits storage from this cycle's report but still sends the // rest). The returned slice is always non-nil so it marshals as [] when empty. func (o *Observer) Observe(ctx context.Context) ([]hub.StorageTarget, error) { snap, err := o.snapshot(ctx) if err != nil { return nil, err } out := make([]hub.StorageTarget, 0, len(snap)) for _, s := range snap { out = append(out, o.enrich(ctx, s)) } return out, nil } // enrich adds the PRIVILEGED reads (SMART, thin-pool metadata) on top of the base target. // Only Observe calls this (the watchdog's Known path skips it — these are the slow, // root-shelling reads). A nil ops or a per-target failure degrades gracefully: SMART stays // UNKNOWN, metadata stays null. func (o *Observer) enrich(ctx context.Context, ob observed) hub.StorageTarget { t := ob.target if o.ops == nil { return t } // SMART: only for dir-backed targets with a resolvable whole-disk device. if ob.cat == catDir && t.BackingDevice != "" { if dev, ok := smartDeviceFor(t.BackingDevice); ok { if sm, err := o.ops.SMART(ctx, dev); err != nil { o.logger.Warn("storage: SMART read failed; health UNKNOWN", "device", dev, "err", err) } else { t.Smart = sm } } } // Thin-pool metadata fill (the value Phase A left null): lvs on the vg/pool. if t.Type == hub.StorageTypeLVMThin && t.ThinPool != nil && ob.src.VGName != "" && ob.src.ThinPool != "" { if frac, ok := o.ops.ThinPoolMetadata(ctx, ob.src.VGName, ob.src.ThinPool); ok { t.ThinPool.MetadataUsedFraction = &frac if frac >= thinPoolWarnFraction { o.logger.Warn("storage: lvmthin pool METADATA fill is high (exhaustion corrupts the pool like data exhaustion)", "storage", t.Name, "metadata_used_fraction", frac) } } } return t } // Known projects the snapshot to the watchdog's lightweight KnownTarget set. Same Proxmox // + host reads as Observe — callers that poll it fast should wrap it in a cache (the // watchdog uses CachingKnownTargets). func (o *Observer) Known(ctx context.Context) ([]KnownTarget, error) { snap, err := o.snapshot(ctx) if err != nil { return nil, err } out := make([]KnownTarget, 0, len(snap)) for _, s := range snap { out = append(out, s.known) } return out, nil } // snapshot does the full build: join cluster config + node usage, then derive each // target's identity, state, class hint, and (for lvmthin) thin-pool fill from host reads. func (o *Observer) snapshot(ctx context.Context) ([]observed, error) { if o.api == nil { return nil, fmt.Errorf("storage: no proxmox api configured") } cluster, err := o.api.ListStorage(ctx) if err != nil { return nil, fmt.Errorf("storage: ListStorage: %w", err) } cfgByName := make(map[string]proxmox.Storage, len(cluster)) for _, c := range cluster { cfgByName[c.Storage] = c } nodeStores, err := o.api.NodeStorage(ctx) if err != nil { return nil, fmt.Errorf("storage: NodeStorage: %w", err) } mounts, err := o.host.Mounts() if err != nil { // Host mount read failed: degrade rather than fail the whole report — Proxmox // usage/active is still meaningful; we just lose mount-derived fields. o.logger.Warn("storage: reading mounts failed; mount/device fields degraded", "err", err) mounts = nil } out := make([]observed, 0, len(nodeStores)) for _, ns := range nodeStores { // Overlay the cluster config (server/export/vgname/...) onto the node entry. s := mergeConfig(ns, cfgByName[ns.Storage]) out = append(out, o.build(s, mounts)) } return out, nil } // build derives one observed target from a merged Storage entry + the mount table. func (o *Observer) build(s proxmox.Storage, mounts []Mount) observed { category := categorize(s.Type) // Resolve the backing device + mount path for dir-like targets. var backingDevice, mountPath string var exactMount bool if category == catDir { if dev, mp, ok := exactMountDevice(mounts, s.Path); ok { backingDevice, mountPath, exactMount = dev, mp, true } else if dev, ok := containingMountDevice(mounts, s.Path); ok { backingDevice = dev // for the class hint only; not its own mount } } // Type: distinguish builtin local / removable USB / fixed local-dir within "dir". removable, removableKnown := false, false if category == catDir && backingDevice != "" { removable, removableKnown = o.host.Removable(backingDevice) } typ := reportType(s, category, removable, removableKnown) // durable_id (DR-load-bearing). var uuid string if category == catDir && backingDevice != "" { uuid, _ = o.host.ResolveUUID(backingDevice) } durableID := deriveDurableID(typ, s, backingDevice, uuid) // Reachability + state. reachable := o.reachable(typ, category, s, backingDevice, exactMount) state := hub.StorageStateAttached if !reachable { state = hub.StorageStateDisconnected } // Class hint (rotational; local block-backed only — a HINT, never authoritative). classHint := "" if category == catDir && backingDevice != "" { if rot, ok := o.host.Rotational(backingDevice); ok { if rot { classHint = "slow" } else { classHint = "fast" } } } tgt := hub.StorageTarget{ Name: s.Storage, Type: typ, DurableID: durableID, State: state, Reachable: reachable, TotalBytes: s.Total, UsedBytes: s.Used, AvailBytes: s.Avail, UsedFraction: usedFraction(s), Content: s.Content, MountPath: mountPath, BackingDevice: backingDevice, ClassHint: classHint, Role: "", // hub-owned; not derivable from a Proxmox def (slice 10) Smart: hub.SmartSummary{Health: hub.SmartUnknown}, } // Thin-pool DATA fill: surfaced prominently for lvmthin (metadata fill is Phase B/lvs). if typ == hub.StorageTypeLVMThin { frac := usedFraction(s) tgt.ThinPool = &hub.ThinPoolFill{DataUsedFraction: frac} if frac >= thinPoolWarnFraction { o.logger.Warn("storage: lvmthin pool data fill is high (a full pool corrupts every guest on it)", "storage", s.Storage, "data_used_fraction", frac) } } return observed{ target: tgt, src: s, cat: category, known: KnownTarget{ Name: s.Storage, Type: typ, DurableID: durableID, UUID: uuid, Network: category == catNetwork, MountBacked: typ == hub.StorageTypeUSB || typ == hub.StorageTypeLocalDir, BackingDevice: backingDevice, MountPath: s.Path, ReachEndpoint: reachEndpoint(typ, s), }, } } // smartDeviceFor maps a backing device (possibly a partition) to its whole-disk path for // smartctl (which targets the disk, not the partition). Returns ok=false when the result // isn't a recognized raw disk (e.g. device-mapper / LVM), so SMART is simply skipped. func smartDeviceFor(device string) (string, bool) { dev := device if m := reNVMePart.FindStringSubmatch(device); m != nil { dev = m[1] // /dev/nvme0n1p2 -> /dev/nvme0n1 } else if m := reSDPart.FindStringSubmatch(device); m != nil { dev = m[1] // /dev/sdb1 -> /dev/sdb } if ValidateSMARTDevice(dev) != nil { return "", false } return dev, true } var ( reNVMePart = regexp.MustCompile(`^(/dev/nvme[0-9]+n[0-9]+)p[0-9]+$`) reSDPart = regexp.MustCompile(`^(/dev/(?:sd|hd|vd)[a-z]+)[0-9]+$`) ) // reachable decides whether the target is currently usable. // - usb / local-dir: a Felhom extra/removable dir storage is realized as its OWN // mountpoint, so reachable = it is currently an exact mount AND its device node exists. // Not-its-own-mount = unplugged/unmounted = disconnected. This is the fast USB-drop // signal — we deliberately do NOT fall through to PVE's active flag, because the // mountpoint directory still exists on the root fs when the device is gone, so active // can read stale-attached. // - local (builtin PVE "local"): lives within the root fs by design, so trust active. // - network (nfs/cifs/pbs) and block-pool (lvmthin/lvm): trust PVE's active flag — PVE // actively probes these and flips active=0 when down. func (o *Observer) reachable(typ string, category storageCategory, s proxmox.Storage, backingDevice string, exactMount bool) bool { switch typ { case hub.StorageTypeUSB, hub.StorageTypeLocalDir: return exactMount && (backingDevice == "" || o.host.DeviceExists(backingDevice)) default: // local, lvmthin, lvm, nfs, cifs, pbs. _ = category return s.Active == 1 } } // usedFraction prefers Proxmox's reported used_fraction, falling back to used/total. func usedFraction(s proxmox.Storage) float64 { if s.UsedFraction > 0 { return s.UsedFraction } if s.Total > 0 { return float64(s.Used) / float64(s.Total) } return 0 } // storageCategory groups Proxmox storage types by how state/identity are derived. type storageCategory int const ( catDir storageCategory = iota // dir-backed (local/usb/local-dir) catNetwork // nfs/cifs/pbs catBlock // lvmthin/lvm catOther ) func categorize(pxType string) storageCategory { switch pxType { case "dir": return catDir case "nfs", "cifs", "smb", "pbs": return catNetwork case "lvmthin", "lvm": return catBlock default: return catOther } } // reportType maps a Proxmox storage type to the reported vocabulary, splitting "dir" into // builtin local / removable usb / fixed local-dir. func reportType(s proxmox.Storage, category storageCategory, removable, removableKnown bool) string { switch category { case catDir: if s.Storage == "local" { return hub.StorageTypeLocal } if removableKnown && removable { return hub.StorageTypeUSB } return hub.StorageTypeLocalDir case catNetwork: switch s.Type { case "nfs": return hub.StorageTypeNFS case "cifs", "smb": return hub.StorageTypeCIFS case "pbs": return hub.StorageTypePBS } case catBlock: if s.Type == "lvmthin" { return hub.StorageTypeLVMThin } return s.Type // "lvm" (thick) passes through } return s.Type } // reachEndpoint builds the host:port the watchdog dials for a network target's // reachability check (default ports per protocol). "" for non-network targets. func reachEndpoint(typ string, s proxmox.Storage) string { if s.Server == "" { return "" } switch typ { case hub.StorageTypeNFS: return netJoin(s.Server, "2049") case hub.StorageTypeCIFS: return netJoin(s.Server, "445") case hub.StorageTypePBS: return netJoin(s.Server, "8007") } return "" } func netJoin(host, port string) string { if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") { host = "[" + host + "]" // IPv6 literal } return host + ":" + port } // exactMountDevice finds the mount whose mountpoint EXACTLY equals path (the target is its // own mount — the meaningful state for a USB/extra disk). func exactMountDevice(mounts []Mount, path string) (device, mountPoint string, ok bool) { if path == "" { return "", "", false } clean := cleanMountPath(path) for _, m := range mounts { if cleanMountPath(m.MountPoint) == clean { return m.Device, m.MountPoint, true } } return "", "", false } // containingMountDevice finds the device of the longest mountpoint that is a prefix of // path (the filesystem that path lives on) — used only for the class-hint disk lookup. func containingMountDevice(mounts []Mount, path string) (device string, ok bool) { if path == "" { return "", false } clean := cleanMountPath(path) best := -1 for _, m := range mounts { mp := cleanMountPath(m.MountPoint) if clean == mp || strings.HasPrefix(clean, mp+"/") || mp == "/" { if len(mp) > best { best, device, ok = len(mp), m.Device, true } } } return device, ok } func cleanMountPath(p string) string { p = strings.TrimRight(p, "/") if p == "" { return "/" } return p } // mergeConfig overlays the cluster-def config fields (which the per-node entry may omit) // onto a node-storage entry, keeping the node's live usage/active values. func mergeConfig(node, cluster proxmox.Storage) proxmox.Storage { if cluster.Storage == "" { return node } if node.Type == "" { node.Type = cluster.Type } if node.Server == "" { node.Server = cluster.Server } if node.Export == "" { node.Export = cluster.Export } if node.Share == "" { node.Share = cluster.Share } if node.Datastore == "" { node.Datastore = cluster.Datastore } if node.Fingerprint == "" { node.Fingerprint = cluster.Fingerprint } if node.VGName == "" { node.VGName = cluster.VGName } if node.ThinPool == "" { node.ThinPool = cluster.ThinPool } if node.Path == "" { node.Path = cluster.Path } if node.Content == "" { node.Content = cluster.Content } return node }