v0.5.0-rc1: slice 5 Phase A — storage observe/report + watchdog (read-only, live)
Fill the slice-3 storage_targets stub and add the fast-poll storage watchdog. Read-only this phase; the host-root surface (mounts/SMART/grow/destructive gate) is Phase B. Hub-owned desired manifest is slice 10, so reconcile against it is built-but-unfed. - internal/storage: StorageTarget wire contract, durable_id derivation per type, HostReader seam (procfs/sysfs, root-free), Observer (storage_targets from ListStorage/NodeStorage + host reads, lvmthin thin-pool fill), and the watchdog (third daemon goroutine; debounced out-of-band report on a known target's attach/disconnect transition). - proxmox.Storage: additive parse-only config fields (durable_id sources). - collector StorageObserver seam; Loop.SetTrigger out-of-band report; daemon runs the watchdog as a third goroutine; StorageConfig knobs. - cross-repo golden kept byte-identical with felhom.eu/hub; bidirectional key-set test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
// Package storage observes and reconciles the host's storage targets (doc 03 §7).
|
||||
//
|
||||
// Slice 5 builds the full model + reconcile machinery; only the read-only, no-hub-desired-
|
||||
// state parts run live:
|
||||
//
|
||||
// - Observe every Proxmox storage target and report it into the host-report
|
||||
// (hub.StorageTarget). The reported view is what the agent SEES; the hub holds the
|
||||
// authoritative manifest (desired class/role/policy/creds) and is not served until
|
||||
// slice 10. So class is a rotational HINT here, never authoritative.
|
||||
// - A storage watchdog: a fast-poll loop that detects a KNOWN target going
|
||||
// attached↔disconnected in seconds and triggers an immediate, debounced out-of-band
|
||||
// host-report (rather than waiting for the slow ~15-minute cycle).
|
||||
//
|
||||
// Phase A (this file set) is read-only: every host read it needs — /proc/mounts,
|
||||
// /dev/disk/by-uuid, /sys/.../rotational, device presence — is non-privileged. Anything
|
||||
// needing root (SMART via smartctl, lvs for thin-pool metadata, blkid) is deferred to
|
||||
// Phase B's privileged HostOps surface.
|
||||
//
|
||||
// Layout:
|
||||
// - hostread.go — the HostReader seam + a non-privileged procfs/sysfs implementation.
|
||||
// - durableid.go — deterministic durable_id derivation per target type (the
|
||||
// DR-load-bearing field: the hub re-attaches the RIGHT drive by it).
|
||||
// - observe.go — the Observer: builds []hub.StorageTarget from Proxmox + host reads.
|
||||
// - watchdog.go — the fast-poll watchdog: transition detection + debounced trigger.
|
||||
//
|
||||
// The collector (internal/hub) calls the Observer through a narrow seam, so hub does not
|
||||
// import storage (storage imports hub for the wire type) — the same interface-seam pattern
|
||||
// the collector uses for proxmox and cloudflared.
|
||||
package storage
|
||||
@@ -0,0 +1,80 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// deriveDurableID computes the DR-load-bearing durable identifier for a target (doc 03
|
||||
// §7). It MUST be deterministic: the hub stores it and, on host loss, re-attaches the
|
||||
// RIGHT physical target by it — the false-id failure mode is re-attaching the WRONG disk.
|
||||
//
|
||||
// Per type:
|
||||
// - usb / local-dir: the filesystem UUID of the backing device (survives re-cabling /
|
||||
// re-enumeration that renames /dev/sdX).
|
||||
// - nfs / cifs: "server:export" (or "server:share") — the network identity.
|
||||
// - pbs: "server:datastore" plus the cert fingerprint ("…#<fp>") — the repo
|
||||
// identity (the fingerprint pins WHICH PBS, so a spoofed server is a different id).
|
||||
// - lvmthin / lvm: "vgname/thinpool" (or "vgname") — informational but stable; the VG
|
||||
// is local and not re-attached cross-host, so a stable name is enough.
|
||||
// - local (builtin): the backing fs UUID if resolvable, else the path — informational.
|
||||
//
|
||||
// uuid is the already-resolved backing-device UUID ("" when unresolved); the caller
|
||||
// resolves it once (it also needs it for nothing else, so we pass it in to avoid a second
|
||||
// by-uuid scan).
|
||||
func deriveDurableID(typ string, s proxmox.Storage, backingDevice, uuid string) string {
|
||||
switch typ {
|
||||
case hubTypeNFS:
|
||||
if s.Server != "" && s.Export != "" {
|
||||
return s.Server + ":" + s.Export
|
||||
}
|
||||
case hubTypeCIFS:
|
||||
if s.Server != "" && s.Share != "" {
|
||||
return s.Server + ":" + s.Share
|
||||
}
|
||||
case hubTypePBS:
|
||||
repo := s.Datastore
|
||||
if s.Server != "" {
|
||||
repo = s.Server + ":" + s.Datastore
|
||||
}
|
||||
if repo != "" {
|
||||
if s.Fingerprint != "" {
|
||||
return repo + "#" + strings.ToLower(s.Fingerprint)
|
||||
}
|
||||
return repo
|
||||
}
|
||||
case hubTypeLVMThin, "lvm":
|
||||
if s.VGName != "" {
|
||||
if s.ThinPool != "" {
|
||||
return s.VGName + "/" + s.ThinPool
|
||||
}
|
||||
return s.VGName
|
||||
}
|
||||
case hubTypeUSB, hubTypeLocalDir, hubTypeLocal:
|
||||
if uuid != "" {
|
||||
return "uuid:" + uuid
|
||||
}
|
||||
if backingDevice != "" {
|
||||
return "dev:" + backingDevice
|
||||
}
|
||||
if s.Path != "" {
|
||||
return "path:" + s.Path
|
||||
}
|
||||
}
|
||||
// Fallback: a stable, unambiguous id from the storage name — never empty (an empty
|
||||
// durable_id would defeat the hub's re-attach lookup).
|
||||
return "store:" + s.Storage
|
||||
}
|
||||
|
||||
// Reported storage-type strings (mirror hub's StorageType* constants without importing
|
||||
// hub here for the bare strings — the observer maps to these).
|
||||
const (
|
||||
hubTypeLocalDir = "local-dir"
|
||||
hubTypeLVMThin = "lvmthin"
|
||||
hubTypeUSB = "usb"
|
||||
hubTypeNFS = "nfs"
|
||||
hubTypeCIFS = "cifs"
|
||||
hubTypePBS = "pbs"
|
||||
hubTypeLocal = "local"
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// durable_id is the DR-load-bearing field — the hub re-attaches the RIGHT physical target
|
||||
// by it. Each type must derive deterministically; the false-id failure mode is
|
||||
// re-attaching the WRONG disk, so this table pins the per-type shape.
|
||||
func TestDeriveDurableID(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
typ string
|
||||
s proxmox.Storage
|
||||
backingDevice string
|
||||
uuid string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "usb by fs-uuid",
|
||||
typ: hubTypeUSB,
|
||||
s: proxmox.Storage{Storage: "usb-backup", Path: "/mnt/usb-backup"},
|
||||
uuid: "0fc6-abcd", want: "uuid:0fc6-abcd",
|
||||
},
|
||||
{
|
||||
name: "local-dir by fs-uuid",
|
||||
typ: hubTypeLocalDir, s: proxmox.Storage{Storage: "extra"}, uuid: "dead-beef",
|
||||
want: "uuid:dead-beef",
|
||||
},
|
||||
{
|
||||
name: "usb falls back to device when uuid unresolved",
|
||||
typ: hubTypeUSB, s: proxmox.Storage{Storage: "usb-backup"}, backingDevice: "/dev/sdb1",
|
||||
want: "dev:/dev/sdb1",
|
||||
},
|
||||
{
|
||||
name: "nfs server:export",
|
||||
typ: hubTypeNFS, s: proxmox.Storage{Storage: "nfs-arch", Server: "10.0.0.5", Export: "/export/b"},
|
||||
want: "10.0.0.5:/export/b",
|
||||
},
|
||||
{
|
||||
name: "cifs server:share",
|
||||
typ: hubTypeCIFS, s: proxmox.Storage{Storage: "cifs", Server: "nas.local", Share: "backups"},
|
||||
want: "nas.local:backups",
|
||||
},
|
||||
{
|
||||
name: "pbs repo + fingerprint (lowercased)",
|
||||
typ: hubTypePBS,
|
||||
s: proxmox.Storage{Storage: "pbs", Server: "pbs.local", Datastore: "store1", Fingerprint: "AB:CD:EF"},
|
||||
want: "pbs.local:store1#ab:cd:ef",
|
||||
},
|
||||
{
|
||||
name: "pbs without fingerprint",
|
||||
typ: hubTypePBS, s: proxmox.Storage{Storage: "pbs", Server: "pbs.local", Datastore: "store1"},
|
||||
want: "pbs.local:store1",
|
||||
},
|
||||
{
|
||||
name: "lvmthin vg/pool",
|
||||
typ: hubTypeLVMThin, s: proxmox.Storage{Storage: "local-lvm", VGName: "pve", ThinPool: "data"},
|
||||
want: "pve/data",
|
||||
},
|
||||
{
|
||||
name: "lvm (thick) vg only",
|
||||
typ: "lvm", s: proxmox.Storage{Storage: "vg0", VGName: "vg0"},
|
||||
want: "vg0",
|
||||
},
|
||||
{
|
||||
name: "local builtin by path when no uuid",
|
||||
typ: hubTypeLocal, s: proxmox.Storage{Storage: "local", Path: "/var/lib/vz"},
|
||||
want: "path:/var/lib/vz",
|
||||
},
|
||||
{
|
||||
name: "unknown/unresolvable falls back to store name (never empty)",
|
||||
typ: hubTypeNFS, s: proxmox.Storage{Storage: "broken-nfs"}, // missing server/export
|
||||
want: "store:broken-nfs",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := deriveDurableID(c.typ, c.s, c.backingDevice, c.uuid)
|
||||
if got != c.want {
|
||||
t.Errorf("deriveDurableID = %q, want %q", got, c.want)
|
||||
}
|
||||
if got == "" {
|
||||
t.Error("durable_id must never be empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HostReader is the non-privileged host-read seam the observer and watchdog need. All of
|
||||
// it is read-only and root-free: the active mount table, fs-UUID resolution via the
|
||||
// /dev/disk/by-uuid symlinks, block-device presence, and the rotational/removable sysfs
|
||||
// flags. Production is *ProcHostReader; tests inject a fake.
|
||||
//
|
||||
// Anything that needs root (smartctl, lvs, blkid) is NOT here — it lands on Phase B's
|
||||
// privileged HostOps surface. Keep this seam root-free.
|
||||
type HostReader interface {
|
||||
// Mounts parses the active mount table (/proc/mounts).
|
||||
Mounts() ([]Mount, error)
|
||||
// ResolveUUID returns the filesystem UUID of a block device, derived from the
|
||||
// /dev/disk/by-uuid symlinks. ok=false when the device has no by-uuid entry.
|
||||
ResolveUUID(device string) (uuid string, ok bool)
|
||||
// DeviceExists reports whether a block-device node is present (the fast USB-drop
|
||||
// signal for the watchdog).
|
||||
DeviceExists(device string) bool
|
||||
// Rotational reads the backing disk's rotational flag (true=HDD/slow, false=SSD/fast).
|
||||
// ok=false when it cannot be determined (network fs, missing sysfs, device-mapper).
|
||||
Rotational(device string) (rotational bool, ok bool)
|
||||
// Removable reads the backing disk's removable flag (true => a USB/hot-plug device).
|
||||
// ok=false when it cannot be determined.
|
||||
Removable(device string) (removable bool, ok bool)
|
||||
}
|
||||
|
||||
// Mount is one active-mount-table entry.
|
||||
type Mount struct {
|
||||
Device string // e.g. "/dev/sdb1", "server:/export", "//server/share"
|
||||
MountPoint string
|
||||
FSType string
|
||||
}
|
||||
|
||||
// ProcHostReader is the production HostReader: it reads the host's /proc, /dev, and /sys.
|
||||
// The paths are fields so tests CAN point it at fixtures, though most tests use a fully
|
||||
// fake HostReader instead.
|
||||
type ProcHostReader struct {
|
||||
ProcMounts string // default "/proc/mounts"
|
||||
ByUUIDDir string // default "/dev/disk/by-uuid"
|
||||
SysClass string // default "/sys/class/block"
|
||||
}
|
||||
|
||||
// NewProcHostReader builds a ProcHostReader with the standard host paths.
|
||||
func NewProcHostReader() *ProcHostReader {
|
||||
return &ProcHostReader{
|
||||
ProcMounts: "/proc/mounts",
|
||||
ByUUIDDir: "/dev/disk/by-uuid",
|
||||
SysClass: "/sys/class/block",
|
||||
}
|
||||
}
|
||||
|
||||
// Mounts parses /proc/mounts. The format is space-separated, octal-escaped fields:
|
||||
// device mountpoint fstype options dump pass. We unescape the first three.
|
||||
func (r *ProcHostReader) Mounts() ([]Mount, error) {
|
||||
f, err := os.Open(r.procMounts())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var out []Mount
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
out = append(out, Mount{
|
||||
Device: unescapeMount(fields[0]),
|
||||
MountPoint: unescapeMount(fields[1]),
|
||||
FSType: fields[2],
|
||||
})
|
||||
}
|
||||
return out, sc.Err()
|
||||
}
|
||||
|
||||
// ResolveUUID reverse-maps a device path to its fs-UUID by reading the /dev/disk/by-uuid
|
||||
// symlinks and matching the canonical target of each against the device.
|
||||
func (r *ProcHostReader) ResolveUUID(device string) (string, bool) {
|
||||
if device == "" {
|
||||
return "", false
|
||||
}
|
||||
want := canonPath(device)
|
||||
entries, err := os.ReadDir(r.byUUIDDir())
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, e := range entries {
|
||||
link := filepath.Join(r.byUUIDDir(), e.Name())
|
||||
target, err := os.Readlink(link)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !filepath.IsAbs(target) {
|
||||
target = filepath.Join(r.byUUIDDir(), target)
|
||||
}
|
||||
if canonPath(target) == want {
|
||||
return e.Name(), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// DeviceExists stats the device node (after resolving symlinks like /dev/disk/by-uuid/X).
|
||||
func (r *ProcHostReader) DeviceExists(device string) bool {
|
||||
if device == "" {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(device)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Rotational reads /sys/block/<parent-disk>/queue/rotational for the device's backing
|
||||
// disk. "1" => rotational (HDD/slow), "0" => SSD/fast.
|
||||
func (r *ProcHostReader) Rotational(device string) (bool, bool) {
|
||||
disk, ok := r.parentDisk(device)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(r.sysBlockDir(), disk, "queue", "rotational"))
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
switch strings.TrimSpace(string(b)) {
|
||||
case "1":
|
||||
return true, true
|
||||
case "0":
|
||||
return false, true
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// Removable reads /sys/block/<parent-disk>/removable. "1" => removable (USB/hot-plug).
|
||||
func (r *ProcHostReader) Removable(device string) (bool, bool) {
|
||||
disk, ok := r.parentDisk(device)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(r.sysBlockDir(), disk, "removable"))
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
switch strings.TrimSpace(string(b)) {
|
||||
case "1":
|
||||
return true, true
|
||||
case "0":
|
||||
return false, true
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// parentDisk maps a device path (possibly a partition like /dev/sdb1 or /dev/nvme0n1p2)
|
||||
// to its parent disk's sysfs name (sdb / nvme0n1). It uses /sys/class/block/<name>, whose
|
||||
// real path ends in .../<disk>/<partition> for a partition and .../<disk> for a whole disk.
|
||||
func (r *ProcHostReader) parentDisk(device string) (string, bool) {
|
||||
name := filepath.Base(strings.TrimSpace(device))
|
||||
if name == "" || name == "." || name == "/" {
|
||||
return "", false
|
||||
}
|
||||
real, err := filepath.EvalSymlinks(filepath.Join(r.sysClass(), name))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
// If <name> is a partition, /sys/class/block/<name>/partition exists and its parent
|
||||
// directory is the disk. Otherwise <name> IS the disk.
|
||||
if _, err := os.Stat(filepath.Join(real, "partition")); err == nil {
|
||||
return filepath.Base(filepath.Dir(real)), true
|
||||
}
|
||||
return filepath.Base(real), true
|
||||
}
|
||||
|
||||
// sysBlockDir derives /sys/block from the configured /sys/class/block.
|
||||
func (r *ProcHostReader) sysBlockDir() string {
|
||||
return filepath.Join(filepath.Dir(filepath.Dir(r.sysClass())), "block")
|
||||
}
|
||||
|
||||
func (r *ProcHostReader) procMounts() string {
|
||||
if r.ProcMounts != "" {
|
||||
return r.ProcMounts
|
||||
}
|
||||
return "/proc/mounts"
|
||||
}
|
||||
func (r *ProcHostReader) byUUIDDir() string {
|
||||
if r.ByUUIDDir != "" {
|
||||
return r.ByUUIDDir
|
||||
}
|
||||
return "/dev/disk/by-uuid"
|
||||
}
|
||||
func (r *ProcHostReader) sysClass() string {
|
||||
if r.SysClass != "" {
|
||||
return r.SysClass
|
||||
}
|
||||
return "/sys/class/block"
|
||||
}
|
||||
|
||||
// canonPath resolves symlinks for a best-effort canonical comparison, falling back to the
|
||||
// cleaned path when the target can't be resolved (e.g. the device just disappeared).
|
||||
func canonPath(p string) string {
|
||||
if real, err := filepath.EvalSymlinks(p); err == nil {
|
||||
return real
|
||||
}
|
||||
return filepath.Clean(p)
|
||||
}
|
||||
|
||||
// unescapeMount decodes the octal \040-style escapes /proc/mounts uses for spaces, tabs,
|
||||
// newlines and backslashes in the device/mountpoint fields.
|
||||
func unescapeMount(s string) string {
|
||||
if !strings.Contains(s, `\`) {
|
||||
return s
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+3 < len(s) && isOctal(s[i+1]) && isOctal(s[i+2]) && isOctal(s[i+3]) {
|
||||
v := (int(s[i+1]-'0') << 6) | (int(s[i+2]-'0') << 3) | int(s[i+3]-'0')
|
||||
b.WriteByte(byte(v))
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
b.WriteByte(s[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func isOctal(c byte) bool { return c >= '0' && c <= '7' }
|
||||
@@ -0,0 +1,398 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// thinPoolWarnFraction is the lvmthin DATA-fill level above which the observer logs a
|
||||
// prominent warning. A full thin-pool corrupts EVERY guest on it (the storage analog of a
|
||||
// single-node OOM), so it must be visible early — well before slice-10 policy exists.
|
||||
const thinPoolWarnFraction = 0.85
|
||||
|
||||
// StorageAPI is the read-only Proxmox surface the observer needs. *proxmox.Client
|
||||
// satisfies it. ListStorage (cluster) carries the type-specific config (server/export/
|
||||
// vgname/thinpool/fingerprint) that NodeStorage may omit; NodeStorage carries live usage
|
||||
// + the per-node active flag. The observer joins them by storage name.
|
||||
type StorageAPI interface {
|
||||
Node() string
|
||||
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||
NodeStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||
}
|
||||
|
||||
// Observer builds the observed storage view from Proxmox + non-privileged host reads.
|
||||
type Observer struct {
|
||||
api StorageAPI
|
||||
host HostReader
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewObserver builds an Observer. host defaults to a ProcHostReader; logger to the
|
||||
// default. A nil api makes Observe/Known return an error (misconfiguration), never panic.
|
||||
func NewObserver(api StorageAPI, host HostReader, logger *slog.Logger) *Observer {
|
||||
if host == nil {
|
||||
host = NewProcHostReader()
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Observer{api: api, host: host, logger: logger}
|
||||
}
|
||||
|
||||
// observed is the rich internal view of one target, from which both the reported
|
||||
// hub.StorageTarget and the watchdog's KnownTarget are projected.
|
||||
type observed struct {
|
||||
target hub.StorageTarget
|
||||
known KnownTarget
|
||||
}
|
||||
|
||||
// Observe builds the reported []hub.StorageTarget. A non-nil error means the Proxmox read
|
||||
// failed (the collector then omits storage from this cycle's report but still sends the
|
||||
// rest). The returned slice is always non-nil so it marshals as [] when empty.
|
||||
func (o *Observer) Observe(ctx context.Context) ([]hub.StorageTarget, error) {
|
||||
snap, err := o.snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]hub.StorageTarget, 0, len(snap))
|
||||
for _, s := range snap {
|
||||
out = append(out, s.target)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Known projects the snapshot to the watchdog's lightweight KnownTarget set. Same Proxmox
|
||||
// + host reads as Observe — callers that poll it fast should wrap it in a cache (the
|
||||
// watchdog uses CachingKnownTargets).
|
||||
func (o *Observer) Known(ctx context.Context) ([]KnownTarget, error) {
|
||||
snap, err := o.snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]KnownTarget, 0, len(snap))
|
||||
for _, s := range snap {
|
||||
out = append(out, s.known)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// snapshot does the full build: join cluster config + node usage, then derive each
|
||||
// target's identity, state, class hint, and (for lvmthin) thin-pool fill from host reads.
|
||||
func (o *Observer) snapshot(ctx context.Context) ([]observed, error) {
|
||||
if o.api == nil {
|
||||
return nil, fmt.Errorf("storage: no proxmox api configured")
|
||||
}
|
||||
cluster, err := o.api.ListStorage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: ListStorage: %w", err)
|
||||
}
|
||||
cfgByName := make(map[string]proxmox.Storage, len(cluster))
|
||||
for _, c := range cluster {
|
||||
cfgByName[c.Storage] = c
|
||||
}
|
||||
nodeStores, err := o.api.NodeStorage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: NodeStorage: %w", err)
|
||||
}
|
||||
mounts, err := o.host.Mounts()
|
||||
if err != nil {
|
||||
// Host mount read failed: degrade rather than fail the whole report — Proxmox
|
||||
// usage/active is still meaningful; we just lose mount-derived fields.
|
||||
o.logger.Warn("storage: reading mounts failed; mount/device fields degraded", "err", err)
|
||||
mounts = nil
|
||||
}
|
||||
|
||||
out := make([]observed, 0, len(nodeStores))
|
||||
for _, ns := range nodeStores {
|
||||
// Overlay the cluster config (server/export/vgname/...) onto the node entry.
|
||||
s := mergeConfig(ns, cfgByName[ns.Storage])
|
||||
out = append(out, o.build(s, mounts))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// build derives one observed target from a merged Storage entry + the mount table.
|
||||
func (o *Observer) build(s proxmox.Storage, mounts []Mount) observed {
|
||||
category := categorize(s.Type)
|
||||
|
||||
// Resolve the backing device + mount path for dir-like targets.
|
||||
var backingDevice, mountPath string
|
||||
var exactMount bool
|
||||
if category == catDir {
|
||||
if dev, mp, ok := exactMountDevice(mounts, s.Path); ok {
|
||||
backingDevice, mountPath, exactMount = dev, mp, true
|
||||
} else if dev, ok := containingMountDevice(mounts, s.Path); ok {
|
||||
backingDevice = dev // for the class hint only; not its own mount
|
||||
}
|
||||
}
|
||||
|
||||
// Type: distinguish builtin local / removable USB / fixed local-dir within "dir".
|
||||
removable, removableKnown := false, false
|
||||
if category == catDir && backingDevice != "" {
|
||||
removable, removableKnown = o.host.Removable(backingDevice)
|
||||
}
|
||||
typ := reportType(s, category, removable, removableKnown)
|
||||
|
||||
// durable_id (DR-load-bearing).
|
||||
var uuid string
|
||||
if category == catDir && backingDevice != "" {
|
||||
uuid, _ = o.host.ResolveUUID(backingDevice)
|
||||
}
|
||||
durableID := deriveDurableID(typ, s, backingDevice, uuid)
|
||||
|
||||
// Reachability + state.
|
||||
reachable := o.reachable(typ, category, s, backingDevice, exactMount)
|
||||
state := hub.StorageStateAttached
|
||||
if !reachable {
|
||||
state = hub.StorageStateDisconnected
|
||||
}
|
||||
|
||||
// Class hint (rotational; local block-backed only — a HINT, never authoritative).
|
||||
classHint := ""
|
||||
if category == catDir && backingDevice != "" {
|
||||
if rot, ok := o.host.Rotational(backingDevice); ok {
|
||||
if rot {
|
||||
classHint = "slow"
|
||||
} else {
|
||||
classHint = "fast"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tgt := hub.StorageTarget{
|
||||
Name: s.Storage,
|
||||
Type: typ,
|
||||
DurableID: durableID,
|
||||
State: state,
|
||||
Reachable: reachable,
|
||||
TotalBytes: s.Total,
|
||||
UsedBytes: s.Used,
|
||||
AvailBytes: s.Avail,
|
||||
UsedFraction: usedFraction(s),
|
||||
Content: s.Content,
|
||||
MountPath: mountPath,
|
||||
BackingDevice: backingDevice,
|
||||
ClassHint: classHint,
|
||||
Role: "", // hub-owned; not derivable from a Proxmox def (slice 10)
|
||||
Smart: hub.SmartSummary{Health: hub.SmartUnknown},
|
||||
}
|
||||
|
||||
// Thin-pool DATA fill: surfaced prominently for lvmthin (metadata fill is Phase B/lvs).
|
||||
if typ == hub.StorageTypeLVMThin {
|
||||
frac := usedFraction(s)
|
||||
tgt.ThinPool = &hub.ThinPoolFill{DataUsedFraction: frac}
|
||||
if frac >= thinPoolWarnFraction {
|
||||
o.logger.Warn("storage: lvmthin pool data fill is high (a full pool corrupts every guest on it)",
|
||||
"storage", s.Storage, "data_used_fraction", frac)
|
||||
}
|
||||
}
|
||||
|
||||
return observed{
|
||||
target: tgt,
|
||||
known: KnownTarget{
|
||||
Name: s.Storage,
|
||||
Type: typ,
|
||||
DurableID: durableID,
|
||||
Network: category == catNetwork,
|
||||
MountBacked: typ == hub.StorageTypeUSB || typ == hub.StorageTypeLocalDir,
|
||||
BackingDevice: backingDevice,
|
||||
MountPath: s.Path,
|
||||
ReachEndpoint: reachEndpoint(typ, s),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// reachable decides whether the target is currently usable.
|
||||
// - usb / local-dir: a Felhom extra/removable dir storage is realized as its OWN
|
||||
// mountpoint, so reachable = it is currently an exact mount AND its device node exists.
|
||||
// Not-its-own-mount = unplugged/unmounted = disconnected. This is the fast USB-drop
|
||||
// signal — we deliberately do NOT fall through to PVE's active flag, because the
|
||||
// mountpoint directory still exists on the root fs when the device is gone, so active
|
||||
// can read stale-attached.
|
||||
// - local (builtin PVE "local"): lives within the root fs by design, so trust active.
|
||||
// - network (nfs/cifs/pbs) and block-pool (lvmthin/lvm): trust PVE's active flag — PVE
|
||||
// actively probes these and flips active=0 when down.
|
||||
func (o *Observer) reachable(typ string, category storageCategory, s proxmox.Storage, backingDevice string, exactMount bool) bool {
|
||||
switch typ {
|
||||
case hub.StorageTypeUSB, hub.StorageTypeLocalDir:
|
||||
return exactMount && (backingDevice == "" || o.host.DeviceExists(backingDevice))
|
||||
default:
|
||||
// local, lvmthin, lvm, nfs, cifs, pbs.
|
||||
_ = category
|
||||
return s.Active == 1
|
||||
}
|
||||
}
|
||||
|
||||
// usedFraction prefers Proxmox's reported used_fraction, falling back to used/total.
|
||||
func usedFraction(s proxmox.Storage) float64 {
|
||||
if s.UsedFraction > 0 {
|
||||
return s.UsedFraction
|
||||
}
|
||||
if s.Total > 0 {
|
||||
return float64(s.Used) / float64(s.Total)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// storageCategory groups Proxmox storage types by how state/identity are derived.
|
||||
type storageCategory int
|
||||
|
||||
const (
|
||||
catDir storageCategory = iota // dir-backed (local/usb/local-dir)
|
||||
catNetwork // nfs/cifs/pbs
|
||||
catBlock // lvmthin/lvm
|
||||
catOther
|
||||
)
|
||||
|
||||
func categorize(pxType string) storageCategory {
|
||||
switch pxType {
|
||||
case "dir":
|
||||
return catDir
|
||||
case "nfs", "cifs", "smb", "pbs":
|
||||
return catNetwork
|
||||
case "lvmthin", "lvm":
|
||||
return catBlock
|
||||
default:
|
||||
return catOther
|
||||
}
|
||||
}
|
||||
|
||||
// reportType maps a Proxmox storage type to the reported vocabulary, splitting "dir" into
|
||||
// builtin local / removable usb / fixed local-dir.
|
||||
func reportType(s proxmox.Storage, category storageCategory, removable, removableKnown bool) string {
|
||||
switch category {
|
||||
case catDir:
|
||||
if s.Storage == "local" {
|
||||
return hub.StorageTypeLocal
|
||||
}
|
||||
if removableKnown && removable {
|
||||
return hub.StorageTypeUSB
|
||||
}
|
||||
return hub.StorageTypeLocalDir
|
||||
case catNetwork:
|
||||
switch s.Type {
|
||||
case "nfs":
|
||||
return hub.StorageTypeNFS
|
||||
case "cifs", "smb":
|
||||
return hub.StorageTypeCIFS
|
||||
case "pbs":
|
||||
return hub.StorageTypePBS
|
||||
}
|
||||
case catBlock:
|
||||
if s.Type == "lvmthin" {
|
||||
return hub.StorageTypeLVMThin
|
||||
}
|
||||
return s.Type // "lvm" (thick) passes through
|
||||
}
|
||||
return s.Type
|
||||
}
|
||||
|
||||
// reachEndpoint builds the host:port the watchdog dials for a network target's
|
||||
// reachability check (default ports per protocol). "" for non-network targets.
|
||||
func reachEndpoint(typ string, s proxmox.Storage) string {
|
||||
if s.Server == "" {
|
||||
return ""
|
||||
}
|
||||
switch typ {
|
||||
case hub.StorageTypeNFS:
|
||||
return netJoin(s.Server, "2049")
|
||||
case hub.StorageTypeCIFS:
|
||||
return netJoin(s.Server, "445")
|
||||
case hub.StorageTypePBS:
|
||||
return netJoin(s.Server, "8007")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func netJoin(host, port string) string {
|
||||
if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") {
|
||||
host = "[" + host + "]" // IPv6 literal
|
||||
}
|
||||
return host + ":" + port
|
||||
}
|
||||
|
||||
// exactMountDevice finds the mount whose mountpoint EXACTLY equals path (the target is its
|
||||
// own mount — the meaningful state for a USB/extra disk).
|
||||
func exactMountDevice(mounts []Mount, path string) (device, mountPoint string, ok bool) {
|
||||
if path == "" {
|
||||
return "", "", false
|
||||
}
|
||||
clean := cleanMountPath(path)
|
||||
for _, m := range mounts {
|
||||
if cleanMountPath(m.MountPoint) == clean {
|
||||
return m.Device, m.MountPoint, true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// containingMountDevice finds the device of the longest mountpoint that is a prefix of
|
||||
// path (the filesystem that path lives on) — used only for the class-hint disk lookup.
|
||||
func containingMountDevice(mounts []Mount, path string) (device string, ok bool) {
|
||||
if path == "" {
|
||||
return "", false
|
||||
}
|
||||
clean := cleanMountPath(path)
|
||||
best := -1
|
||||
for _, m := range mounts {
|
||||
mp := cleanMountPath(m.MountPoint)
|
||||
if clean == mp || strings.HasPrefix(clean, mp+"/") || mp == "/" {
|
||||
if len(mp) > best {
|
||||
best, device, ok = len(mp), m.Device, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return device, ok
|
||||
}
|
||||
|
||||
func cleanMountPath(p string) string {
|
||||
p = strings.TrimRight(p, "/")
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// mergeConfig overlays the cluster-def config fields (which the per-node entry may omit)
|
||||
// onto a node-storage entry, keeping the node's live usage/active values.
|
||||
func mergeConfig(node, cluster proxmox.Storage) proxmox.Storage {
|
||||
if cluster.Storage == "" {
|
||||
return node
|
||||
}
|
||||
if node.Type == "" {
|
||||
node.Type = cluster.Type
|
||||
}
|
||||
if node.Server == "" {
|
||||
node.Server = cluster.Server
|
||||
}
|
||||
if node.Export == "" {
|
||||
node.Export = cluster.Export
|
||||
}
|
||||
if node.Share == "" {
|
||||
node.Share = cluster.Share
|
||||
}
|
||||
if node.Datastore == "" {
|
||||
node.Datastore = cluster.Datastore
|
||||
}
|
||||
if node.Fingerprint == "" {
|
||||
node.Fingerprint = cluster.Fingerprint
|
||||
}
|
||||
if node.VGName == "" {
|
||||
node.VGName = cluster.VGName
|
||||
}
|
||||
if node.ThinPool == "" {
|
||||
node.ThinPool = cluster.ThinPool
|
||||
}
|
||||
if node.Path == "" {
|
||||
node.Path = cluster.Path
|
||||
}
|
||||
if node.Content == "" {
|
||||
node.Content = cluster.Content
|
||||
}
|
||||
return node
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
func quietLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
// fakeStorageAPI serves fixed cluster + node storage lists.
|
||||
type fakeStorageAPI struct {
|
||||
node string
|
||||
cluster []proxmox.Storage
|
||||
nodeSt []proxmox.Storage
|
||||
listErr error
|
||||
nodeErr error
|
||||
}
|
||||
|
||||
func (f *fakeStorageAPI) Node() string { return f.node }
|
||||
func (f *fakeStorageAPI) ListStorage(context.Context) ([]proxmox.Storage, error) {
|
||||
return f.cluster, f.listErr
|
||||
}
|
||||
func (f *fakeStorageAPI) NodeStorage(context.Context) ([]proxmox.Storage, error) {
|
||||
return f.nodeSt, f.nodeErr
|
||||
}
|
||||
|
||||
// fakeHostReader is a fully synthetic HostReader — no real devices touched.
|
||||
type fakeHostReader struct {
|
||||
mounts []Mount
|
||||
mountsErr error
|
||||
uuids map[string]string // device -> uuid
|
||||
exists map[string]bool // device -> present
|
||||
rotational map[string]bool // device -> rotational (presence => known)
|
||||
removable map[string]bool // device -> removable (presence => known)
|
||||
}
|
||||
|
||||
func (h *fakeHostReader) Mounts() ([]Mount, error) { return h.mounts, h.mountsErr }
|
||||
func (h *fakeHostReader) ResolveUUID(device string) (string, bool) {
|
||||
u, ok := h.uuids[device]
|
||||
return u, ok
|
||||
}
|
||||
func (h *fakeHostReader) DeviceExists(device string) bool { return h.exists[device] }
|
||||
func (h *fakeHostReader) Rotational(device string) (bool, bool) {
|
||||
v, ok := h.rotational[device]
|
||||
return v, ok
|
||||
}
|
||||
func (h *fakeHostReader) Removable(device string) (bool, bool) {
|
||||
v, ok := h.removable[device]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// byName indexes observed targets for assertions.
|
||||
func byName(targets []hub.StorageTarget) map[string]hub.StorageTarget {
|
||||
m := make(map[string]hub.StorageTarget, len(targets))
|
||||
for _, t := range targets {
|
||||
m[t.Name] = t
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestObserve_BuildsTargetsFromProxmoxAndHostReads(t *testing.T) {
|
||||
api := &fakeStorageAPI{
|
||||
node: "demo-felhom",
|
||||
cluster: []proxmox.Storage{
|
||||
{Storage: "local", Type: "dir", Content: "vztmpl,backup", Path: "/var/lib/vz"},
|
||||
{Storage: "local-lvm", Type: "lvmthin", Content: "rootdir,images", VGName: "pve", ThinPool: "data"},
|
||||
{Storage: "usb-backup", Type: "dir", Content: "backup", Path: "/mnt/usb-backup"},
|
||||
{Storage: "nfs-arch", Type: "nfs", Content: "backup", Server: "10.0.0.5", Export: "/export/backups"},
|
||||
},
|
||||
nodeSt: []proxmox.Storage{
|
||||
{Storage: "local", Type: "dir", Content: "vztmpl,backup", Path: "/var/lib/vz", Total: 100, Used: 20, Avail: 80, Active: 1, UsedFraction: 0.2},
|
||||
{Storage: "local-lvm", Type: "lvmthin", Content: "rootdir,images", Total: 1000, Used: 900, Avail: 100, Active: 1, UsedFraction: 0.9},
|
||||
{Storage: "usb-backup", Type: "dir", Content: "backup", Path: "/mnt/usb-backup", Total: 2000, Used: 500, Avail: 1500, Active: 1, UsedFraction: 0.25},
|
||||
{Storage: "nfs-arch", Type: "nfs", Content: "backup", Total: 5000, Used: 1000, Avail: 4000, Active: 1, UsedFraction: 0.2},
|
||||
},
|
||||
}
|
||||
host := &fakeHostReader{
|
||||
mounts: []Mount{
|
||||
{Device: "/dev/mapper/pve-root", MountPoint: "/", FSType: "ext4"},
|
||||
{Device: "/dev/sdb1", MountPoint: "/mnt/usb-backup", FSType: "ext4"},
|
||||
},
|
||||
uuids: map[string]string{"/dev/sdb1": "1111-2222"},
|
||||
exists: map[string]bool{"/dev/sdb1": true, "/dev/mapper/pve-root": true},
|
||||
rotational: map[string]bool{"/dev/sdb1": true, "/dev/mapper/pve-root": false},
|
||||
removable: map[string]bool{"/dev/sdb1": true, "/dev/mapper/pve-root": false},
|
||||
}
|
||||
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Observe: %v", err)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("got %d targets, want 4", len(got))
|
||||
}
|
||||
m := byName(got)
|
||||
|
||||
// builtin local: dir within root → type "local", attached via active flag.
|
||||
if local := m["local"]; local.Type != hub.StorageTypeLocal || local.State != hub.StorageStateAttached {
|
||||
t.Errorf("local = %+v, want type=local state=attached", local)
|
||||
}
|
||||
|
||||
// lvmthin: thin-pool DATA fill surfaced; durable_id = vg/pool; no mount/device.
|
||||
lvm := m["local-lvm"]
|
||||
if lvm.Type != hub.StorageTypeLVMThin || lvm.DurableID != "pve/data" {
|
||||
t.Errorf("local-lvm type/durable = %q/%q", lvm.Type, lvm.DurableID)
|
||||
}
|
||||
if lvm.ThinPool == nil || lvm.ThinPool.DataUsedFraction != 0.9 {
|
||||
t.Errorf("local-lvm thin_pool = %+v, want data_used_fraction=0.9", lvm.ThinPool)
|
||||
}
|
||||
if lvm.ThinPool.MetadataUsedFraction != nil {
|
||||
t.Errorf("metadata fill must be nil in Phase A (lvs is Phase B)")
|
||||
}
|
||||
|
||||
// usb: removable dir, mounted → type usb, durable_id from UUID, class_hint slow (rotational).
|
||||
usb := m["usb-backup"]
|
||||
if usb.Type != hub.StorageTypeUSB {
|
||||
t.Errorf("usb-backup type = %q, want usb", usb.Type)
|
||||
}
|
||||
if usb.DurableID != "uuid:1111-2222" {
|
||||
t.Errorf("usb-backup durable_id = %q, want uuid:1111-2222", usb.DurableID)
|
||||
}
|
||||
if usb.ClassHint != "slow" {
|
||||
t.Errorf("usb-backup class_hint = %q, want slow (rotational)", usb.ClassHint)
|
||||
}
|
||||
if usb.MountPath != "/mnt/usb-backup" || usb.BackingDevice != "/dev/sdb1" {
|
||||
t.Errorf("usb-backup mount/device = %q/%q", usb.MountPath, usb.BackingDevice)
|
||||
}
|
||||
if usb.State != hub.StorageStateAttached || !usb.Reachable {
|
||||
t.Errorf("usb-backup should be attached+reachable: %+v", usb)
|
||||
}
|
||||
if usb.ThinPool != nil {
|
||||
t.Errorf("non-lvmthin must omit thin_pool")
|
||||
}
|
||||
|
||||
// nfs: durable_id = server:export; attached via active flag; no class hint.
|
||||
nfs := m["nfs-arch"]
|
||||
if nfs.Type != hub.StorageTypeNFS || nfs.DurableID != "10.0.0.5:/export/backups" {
|
||||
t.Errorf("nfs-arch type/durable = %q/%q", nfs.Type, nfs.DurableID)
|
||||
}
|
||||
if nfs.ClassHint != "" {
|
||||
t.Errorf("network target must have no class hint, got %q", nfs.ClassHint)
|
||||
}
|
||||
|
||||
// SMART is UNKNOWN in Phase A for every target.
|
||||
for _, tgt := range got {
|
||||
if tgt.Smart.Health != hub.SmartUnknown {
|
||||
t.Errorf("%s smart health = %q, want UNKNOWN in Phase A", tgt.Name, tgt.Smart.Health)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserve_USBUnpluggedIsDisconnected(t *testing.T) {
|
||||
api := &fakeStorageAPI{
|
||||
node: "demo-felhom",
|
||||
cluster: []proxmox.Storage{
|
||||
{Storage: "usb-backup", Type: "dir", Content: "backup", Path: "/mnt/usb-backup"},
|
||||
},
|
||||
nodeSt: []proxmox.Storage{
|
||||
// PVE may still show the storage entry (active possibly 1) but the mount is gone.
|
||||
{Storage: "usb-backup", Type: "dir", Content: "backup", Path: "/mnt/usb-backup", Active: 1},
|
||||
},
|
||||
}
|
||||
host := &fakeHostReader{
|
||||
mounts: []Mount{{Device: "/dev/mapper/pve-root", MountPoint: "/", FSType: "ext4"}}, // no /mnt/usb-backup
|
||||
}
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
usb := byName(got)["usb-backup"]
|
||||
// Without its own mount, a USB target is unplugged regardless of PVE's stale active flag.
|
||||
if usb.State != hub.StorageStateDisconnected || usb.Reachable {
|
||||
t.Errorf("unplugged usb should be disconnected/unreachable, got state=%q reachable=%v", usb.State, usb.Reachable)
|
||||
}
|
||||
// Its durable_id falls back to a stable form (no UUID resolvable while detached).
|
||||
if usb.DurableID == "" {
|
||||
t.Errorf("durable_id must never be empty (DR re-attach lookup)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserve_ProxmoxErrorIsFatalForStorage(t *testing.T) {
|
||||
api := &fakeStorageAPI{node: "n", listErr: context.DeadlineExceeded}
|
||||
if _, err := NewObserver(api, &fakeHostReader{}, quietLogger()).Observe(context.Background()); err == nil {
|
||||
t.Fatal("a Proxmox read error must surface (the collector then omits storage this cycle)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserve_MountReadFailureDegradesNotFatal(t *testing.T) {
|
||||
api := &fakeStorageAPI{
|
||||
node: "n",
|
||||
cluster: []proxmox.Storage{{Storage: "local-lvm", Type: "lvmthin", VGName: "pve", ThinPool: "data"}},
|
||||
nodeSt: []proxmox.Storage{{Storage: "local-lvm", Type: "lvmthin", Active: 1, UsedFraction: 0.1}},
|
||||
}
|
||||
host := &fakeHostReader{mountsErr: io.ErrUnexpectedEOF}
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("a host mount-read failure must degrade, not fail: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].DurableID != "pve/data" {
|
||||
t.Errorf("lvmthin still derivable without mounts: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Default watchdog timings (configurable via WatchdogOptions). The poll is FAST (seconds)
|
||||
// so a USB drop is caught in seconds, not at the slow ~15-minute host-report cycle; the
|
||||
// debounce keeps a flapping drive from storming the hub.
|
||||
const (
|
||||
DefaultWatchdogInterval = 8 * time.Second
|
||||
DefaultWatchdogDebounce = 30 * time.Second
|
||||
)
|
||||
|
||||
// KnownTarget is the watchdog's lightweight view of a target it watches. "Known" means a
|
||||
// defined Proxmox storage (and/or a previously-observed-attached one); the watchdog only
|
||||
// flags transitions for targets it has seen — it never reports a never-attached device.
|
||||
type KnownTarget struct {
|
||||
Name string
|
||||
Type string
|
||||
DurableID string
|
||||
Network bool // nfs/cifs/pbs — liveness is a reachability dial, not a device check
|
||||
MountBacked bool // usb/local-dir — a drop = its mountpoint disappears
|
||||
BackingDevice string // resolved block device (local targets)
|
||||
MountPath string // the mountpoint a mount-backed target must occupy
|
||||
ReachEndpoint string // host:port to dial for a network target's reachability
|
||||
}
|
||||
|
||||
// KnownTargets enumerates the currently-known target set. Production wraps the Observer in
|
||||
// CachingKnownTargets so the fast poll doesn't hammer the Proxmox API.
|
||||
type KnownTargets interface {
|
||||
Known(ctx context.Context) ([]KnownTarget, error)
|
||||
}
|
||||
|
||||
// TargetLiveness reports whether one known target is presently up. Production is
|
||||
// HostLiveness (device/mount presence + a reachability dial, all non-privileged); tests
|
||||
// inject a fake.
|
||||
type TargetLiveness interface {
|
||||
Present(ctx context.Context, t KnownTarget) bool
|
||||
}
|
||||
|
||||
// Transition is one observed state change for a known target (for logging/diagnostics).
|
||||
type Transition struct {
|
||||
Name string
|
||||
From string // attached | disconnected
|
||||
To string
|
||||
}
|
||||
|
||||
// Watchdog is the third daemon goroutine (alongside the hub loop + reconcile engine). It
|
||||
// fast-polls the known target set, detects attached↔disconnected transitions, and triggers
|
||||
// an immediate, debounced out-of-band host-report so the hub learns of a drop in seconds.
|
||||
//
|
||||
// It NEVER mutates anything (Phase A is read-only) — the benign re-mount-by-UUID response
|
||||
// to a return lands in Phase B. Here it only observes and signals.
|
||||
type Watchdog struct {
|
||||
targets KnownTargets
|
||||
liveness TargetLiveness
|
||||
interval time.Duration
|
||||
debounce time.Duration
|
||||
trigger func() // request an out-of-band report (debounced by the watchdog)
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
}
|
||||
|
||||
// WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
|
||||
// rest default.
|
||||
type WatchdogOptions struct {
|
||||
Targets KnownTargets
|
||||
Liveness TargetLiveness
|
||||
Trigger func()
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
|
||||
// state, just signals nothing) so it degrades cleanly when no report sink is wired.
|
||||
func NewWatchdog(opts WatchdogOptions) *Watchdog {
|
||||
interval := opts.Interval
|
||||
if interval <= 0 {
|
||||
interval = DefaultWatchdogInterval
|
||||
}
|
||||
debounce := opts.Debounce
|
||||
if debounce <= 0 {
|
||||
debounce = DefaultWatchdogDebounce
|
||||
}
|
||||
logger := opts.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
trigger := opts.Trigger
|
||||
if trigger == nil {
|
||||
trigger = func() {}
|
||||
}
|
||||
return &Watchdog{
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
last: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// Run fast-polls until ctx is cancelled. The first tick establishes the baseline (no
|
||||
// trigger); subsequent ticks detect transitions. Returns nil on ctx cancellation.
|
||||
func (w *Watchdog) Run(ctx context.Context) error {
|
||||
if w.targets == nil || w.liveness == nil {
|
||||
w.logger.Info("storage: watchdog idle (no target source / liveness probe configured)")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
w.logger.Info("storage: watchdog starting", "interval", w.interval, "debounce", w.debounce)
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
w.tick(ctx) // immediate baseline
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.logger.Info("storage: watchdog shutting down", "reason", ctx.Err())
|
||||
return nil
|
||||
case <-t.C:
|
||||
w.tick(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick performs one poll: read the known set, probe each target's liveness, diff against
|
||||
// the last-seen state, and fire a debounced trigger on any transition for a SEEN target.
|
||||
// It is deterministic given w.now — tests drive it directly with a fake clock.
|
||||
func (w *Watchdog) tick(ctx context.Context) {
|
||||
known, err := w.targets.Known(ctx)
|
||||
if err != nil {
|
||||
w.logger.Warn("storage: watchdog could not read known targets; skipping tick", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
var transitions []Transition
|
||||
current := make(map[string]bool, len(known))
|
||||
for _, k := range known {
|
||||
present := w.liveness.Present(ctx, k)
|
||||
current[k.Name] = present
|
||||
prev, seen := w.last[k.Name]
|
||||
if !seen {
|
||||
continue // first observation → baseline only (never flag a never-attached drop)
|
||||
}
|
||||
if prev != present {
|
||||
transitions = append(transitions, Transition{Name: k.Name, From: stateStr(prev), To: stateStr(present)})
|
||||
}
|
||||
}
|
||||
// Replace the baseline with the current snapshot (targets no longer known drop out).
|
||||
w.last = current
|
||||
|
||||
now := w.now()
|
||||
if len(transitions) > 0 {
|
||||
for _, tr := range transitions {
|
||||
w.logger.Warn("storage: watchdog detected target state change",
|
||||
"target", tr.Name, "from", tr.From, "to", tr.To)
|
||||
}
|
||||
if !w.fired || now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, len(transitions))
|
||||
} else {
|
||||
w.pending = true
|
||||
w.logger.Debug("storage: watchdog debouncing transition", "pending_until", w.lastFire.Add(w.debounce))
|
||||
}
|
||||
return
|
||||
}
|
||||
// No new transition, but a debounced one is pending and the window has elapsed → fire.
|
||||
if w.pending && now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// fire requests the out-of-band report and resets the debounce window. Called under w.mu.
|
||||
func (w *Watchdog) fire(now time.Time, n int) {
|
||||
w.lastFire = now
|
||||
w.fired = true
|
||||
w.pending = false
|
||||
w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", n)
|
||||
w.trigger()
|
||||
}
|
||||
|
||||
func stateStr(present bool) string {
|
||||
if present {
|
||||
return "attached"
|
||||
}
|
||||
return "disconnected"
|
||||
}
|
||||
|
||||
// --- production liveness + a caching known-target source ---
|
||||
|
||||
// HostLiveness is the production TargetLiveness: device + mount presence for local
|
||||
// targets (the fast USB-drop signal) and a short reachability dial for network targets.
|
||||
// All non-privileged.
|
||||
type HostLiveness struct {
|
||||
host HostReader
|
||||
dialTimeout time.Duration
|
||||
dial func(network, address string, timeout time.Duration) (net.Conn, error)
|
||||
}
|
||||
|
||||
// NewHostLiveness builds a HostLiveness over a HostReader. dialTimeout defaults to 3s.
|
||||
func NewHostLiveness(host HostReader, dialTimeout time.Duration) *HostLiveness {
|
||||
if host == nil {
|
||||
host = NewProcHostReader()
|
||||
}
|
||||
if dialTimeout <= 0 {
|
||||
dialTimeout = 3 * time.Second
|
||||
}
|
||||
return &HostLiveness{host: host, dialTimeout: dialTimeout, dial: net.DialTimeout}
|
||||
}
|
||||
|
||||
// Present probes one target without touching Proxmox.
|
||||
func (h *HostLiveness) Present(ctx context.Context, t KnownTarget) bool {
|
||||
if t.Network {
|
||||
if t.ReachEndpoint == "" {
|
||||
return true // can't probe → don't false-alarm; the 15-min cycle uses the active flag
|
||||
}
|
||||
conn, err := h.dial("tcp", t.ReachEndpoint, h.dialTimeout)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
if t.MountBacked {
|
||||
// A mount-backed target (USB / extra disk) is present iff its mountpoint is an
|
||||
// active mount AND the backing device node exists.
|
||||
if !h.mounted(t.MountPath) {
|
||||
return false
|
||||
}
|
||||
return t.BackingDevice == "" || h.host.DeviceExists(t.BackingDevice)
|
||||
}
|
||||
// Non-removable builtin targets (local/lvmthin): treated as present here — they don't
|
||||
// "drop" without the whole host going down, which the heartbeat covers.
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *HostLiveness) mounted(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
mounts, err := h.host.Mounts()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _, ok := exactMountDevice(mounts, path)
|
||||
return ok
|
||||
}
|
||||
|
||||
// CachingKnownTargets wraps a slow KnownTargets source (the Observer, which hits Proxmox)
|
||||
// with a TTL so the fast watchdog poll re-derives the known SET only every ttl, while
|
||||
// still probing liveness every tick. A read error returns the last good set (so a
|
||||
// transient Proxmox blip doesn't blank the watchdog's world).
|
||||
type CachingKnownTargets struct {
|
||||
src KnownTargets
|
||||
ttl time.Duration
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
cached []KnownTarget
|
||||
at time.Time
|
||||
loaded bool
|
||||
}
|
||||
|
||||
// NewCachingKnownTargets wraps src, refreshing at most every ttl (default 60s).
|
||||
func NewCachingKnownTargets(src KnownTargets, ttl time.Duration) *CachingKnownTargets {
|
||||
if ttl <= 0 {
|
||||
ttl = 60 * time.Second
|
||||
}
|
||||
return &CachingKnownTargets{src: src, ttl: ttl, now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
// Known returns the cached set, refreshing it when the TTL has elapsed.
|
||||
func (c *CachingKnownTargets) Known(ctx context.Context) ([]KnownTarget, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
now := c.now()
|
||||
if c.loaded && now.Sub(c.at) < c.ttl {
|
||||
return c.cached, nil
|
||||
}
|
||||
fresh, err := c.src.Known(ctx)
|
||||
if err != nil {
|
||||
if c.loaded {
|
||||
return c.cached, nil // serve stale rather than blank on a transient error
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
c.cached, c.at, c.loaded = fresh, now, true
|
||||
return c.cached, nil
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// staticKnown is a settable KnownTargets fake.
|
||||
type staticKnown struct {
|
||||
mu sync.Mutex
|
||||
targets []KnownTarget
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *staticKnown) Known(context.Context) ([]KnownTarget, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls++
|
||||
return s.targets, s.err
|
||||
}
|
||||
|
||||
// mapLiveness is a settable per-target presence fake.
|
||||
type mapLiveness struct {
|
||||
mu sync.Mutex
|
||||
present map[string]bool
|
||||
}
|
||||
|
||||
func (m *mapLiveness) set(name string, p bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.present[name] = p
|
||||
}
|
||||
func (m *mapLiveness) Present(_ context.Context, t KnownTarget) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.present[t.Name]
|
||||
}
|
||||
|
||||
// newTestWatchdog builds a watchdog with a manual clock and a trigger counter.
|
||||
func newTestWatchdog(known KnownTargets, live TargetLiveness, debounce time.Duration) (*Watchdog, *int, *time.Time) {
|
||||
var fires int
|
||||
clock := time.Unix(1_700_000_000, 0).UTC()
|
||||
w := NewWatchdog(WatchdogOptions{
|
||||
Targets: known,
|
||||
Liveness: live,
|
||||
Trigger: func() { fires++ },
|
||||
Interval: time.Second,
|
||||
Debounce: debounce,
|
||||
Logger: quietLogger(),
|
||||
})
|
||||
w.now = func() time.Time { return clock }
|
||||
return w, &fires, &clock
|
||||
}
|
||||
|
||||
func TestWatchdog_BaselineThenDropTriggers(t *testing.T) {
|
||||
known := &staticKnown{targets: []KnownTarget{{Name: "usb", MountBacked: true}}}
|
||||
live := &mapLiveness{present: map[string]bool{"usb": true}}
|
||||
w, fires, _ := newTestWatchdog(known, live, 30*time.Second)
|
||||
ctx := context.Background()
|
||||
|
||||
w.tick(ctx) // baseline: present, no trigger
|
||||
if *fires != 0 {
|
||||
t.Fatalf("baseline tick must not trigger, fires=%d", *fires)
|
||||
}
|
||||
live.set("usb", false) // drop
|
||||
w.tick(ctx)
|
||||
if *fires != 1 {
|
||||
t.Fatalf("a known target drop must trigger an out-of-band report, fires=%d", *fires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_NeverAttachedNotFlagged(t *testing.T) {
|
||||
// A defined-but-absent target (never seen present) must not be flagged on its absence.
|
||||
known := &staticKnown{targets: []KnownTarget{{Name: "usb", MountBacked: true}}}
|
||||
live := &mapLiveness{present: map[string]bool{"usb": false}}
|
||||
w, fires, _ := newTestWatchdog(known, live, 30*time.Second)
|
||||
ctx := context.Background()
|
||||
|
||||
w.tick(ctx) // baseline absent
|
||||
w.tick(ctx) // still absent
|
||||
if *fires != 0 {
|
||||
t.Fatalf("a never-attached target must not trigger, fires=%d", *fires)
|
||||
}
|
||||
// Now it appears (reconnect) → that IS a transition worth reporting.
|
||||
live.set("usb", true)
|
||||
w.tick(ctx)
|
||||
if *fires != 1 {
|
||||
t.Fatalf("attach transition should trigger, fires=%d", *fires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_DebounceCoalescesFlaps(t *testing.T) {
|
||||
known := &staticKnown{targets: []KnownTarget{{Name: "usb", MountBacked: true}}}
|
||||
live := &mapLiveness{present: map[string]bool{"usb": true}}
|
||||
w, fires, clock := newTestWatchdog(known, live, 30*time.Second)
|
||||
ctx := context.Background()
|
||||
|
||||
w.tick(ctx) // baseline present
|
||||
|
||||
// First drop fires immediately (leading edge).
|
||||
live.set("usb", false)
|
||||
w.tick(ctx)
|
||||
if *fires != 1 {
|
||||
t.Fatalf("first drop should fire, fires=%d", *fires)
|
||||
}
|
||||
|
||||
// Flap within the debounce window: re-attach then drop again — suppressed (pending).
|
||||
*clock = clock.Add(5 * time.Second)
|
||||
live.set("usb", true)
|
||||
w.tick(ctx)
|
||||
*clock = clock.Add(5 * time.Second)
|
||||
live.set("usb", false)
|
||||
w.tick(ctx)
|
||||
if *fires != 1 {
|
||||
t.Fatalf("flaps within the debounce window must be coalesced, fires=%d", *fires)
|
||||
}
|
||||
|
||||
// After the window elapses, the pending change fires (trailing edge), even with no new
|
||||
// transition this tick.
|
||||
*clock = clock.Add(30 * time.Second)
|
||||
w.tick(ctx)
|
||||
if *fires != 2 {
|
||||
t.Fatalf("a pending change must fire once the window elapses, fires=%d", *fires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_ReadErrorSkipsTick(t *testing.T) {
|
||||
known := &staticKnown{err: errors.New("proxmox blip")}
|
||||
live := &mapLiveness{present: map[string]bool{}}
|
||||
w, fires, _ := newTestWatchdog(known, live, time.Second)
|
||||
w.tick(context.Background())
|
||||
if *fires != 0 {
|
||||
t.Fatalf("a known-target read error must not trigger, fires=%d", *fires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_RunBaselinesAndStops(t *testing.T) {
|
||||
// Smoke test of the goroutine wiring under -race: Run establishes a baseline and exits
|
||||
// cleanly on ctx cancellation.
|
||||
known := &staticKnown{targets: []KnownTarget{{Name: "usb", MountBacked: true}}}
|
||||
live := &mapLiveness{present: map[string]bool{"usb": true}}
|
||||
w, _, _ := newTestWatchdog(known, live, time.Second)
|
||||
w.now = func() time.Time { return time.Now().UTC() }
|
||||
w.interval = 5 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(ctx) }()
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned %v, want nil on cancel", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("watchdog did not stop on cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingKnownTargets_RefreshesOnTTL(t *testing.T) {
|
||||
src := &staticKnown{targets: []KnownTarget{{Name: "a"}}}
|
||||
clock := time.Unix(1_700_000_000, 0).UTC()
|
||||
c := NewCachingKnownTargets(src, 60*time.Second)
|
||||
c.now = func() time.Time { return clock }
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := c.Known(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.Known(ctx); err != nil { // within TTL → cached
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.calls != 1 {
|
||||
t.Fatalf("within TTL the source must be hit once, calls=%d", src.calls)
|
||||
}
|
||||
clock = clock.Add(61 * time.Second) // past TTL
|
||||
if _, err := c.Known(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.calls != 2 {
|
||||
t.Fatalf("past TTL the source must refresh, calls=%d", src.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingKnownTargets_ServesStaleOnError(t *testing.T) {
|
||||
src := &staticKnown{targets: []KnownTarget{{Name: "a"}}}
|
||||
clock := time.Unix(1_700_000_000, 0).UTC()
|
||||
c := NewCachingKnownTargets(src, 1*time.Second)
|
||||
c.now = func() time.Time { return clock }
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := c.Known(ctx); err != nil { // prime the cache
|
||||
t.Fatal(err)
|
||||
}
|
||||
clock = clock.Add(2 * time.Second)
|
||||
src.mu.Lock()
|
||||
src.err = errors.New("blip")
|
||||
src.mu.Unlock()
|
||||
got, err := c.Known(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("a transient error must serve stale, got err=%v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Name != "a" {
|
||||
t.Fatalf("stale set not served: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostLiveness_MountBackedPresence(t *testing.T) {
|
||||
host := &fakeHostReader{
|
||||
mounts: []Mount{{Device: "/dev/sdb1", MountPoint: "/mnt/usb", FSType: "ext4"}},
|
||||
exists: map[string]bool{"/dev/sdb1": true},
|
||||
}
|
||||
hl := NewHostLiveness(host, time.Second)
|
||||
tgt := KnownTarget{Name: "usb", MountBacked: true, MountPath: "/mnt/usb", BackingDevice: "/dev/sdb1"}
|
||||
if !hl.Present(context.Background(), tgt) {
|
||||
t.Error("mounted device should be present")
|
||||
}
|
||||
|
||||
// Unmount it: no exact mount entry → absent.
|
||||
host.mounts = []Mount{{Device: "/dev/mapper/root", MountPoint: "/", FSType: "ext4"}}
|
||||
if hl.Present(context.Background(), tgt) {
|
||||
t.Error("unmounted device should be absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostLiveness_NetworkDial(t *testing.T) {
|
||||
hl := NewHostLiveness(&fakeHostReader{}, time.Second)
|
||||
var dialed string
|
||||
hl.dial = func(network, addr string, _ time.Duration) (net.Conn, error) {
|
||||
dialed = addr
|
||||
return nil, errors.New("refused")
|
||||
}
|
||||
tgt := KnownTarget{Name: "nfs", Network: true, ReachEndpoint: "10.0.0.5:2049"}
|
||||
if hl.Present(context.Background(), tgt) {
|
||||
t.Error("a refused dial should report not-present")
|
||||
}
|
||||
if dialed != "10.0.0.5:2049" {
|
||||
t.Errorf("dialed %q, want 10.0.0.5:2049", dialed)
|
||||
}
|
||||
|
||||
// No endpoint to probe → don't false-alarm (the slow cycle uses the active flag).
|
||||
if !hl.Present(context.Background(), KnownTarget{Name: "x", Network: true}) {
|
||||
t.Error("network target without endpoint must not be flagged down")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user