v0.5.0-rc1: slice 5 Phase A — storage observe/report + watchdog (read-only, live)

Fill the slice-3 storage_targets stub and add the fast-poll storage watchdog. Read-only this phase; the host-root surface (mounts/SMART/grow/destructive gate) is Phase B. Hub-owned desired manifest is slice 10, so reconcile against it is built-but-unfed. - internal/storage: StorageTarget wire contract, durable_id derivation per type, HostReader seam (procfs/sysfs, root-free), Observer (storage_targets from ListStorage/NodeStorage + host reads, lvmthin thin-pool fill), and the watchdog (third daemon goroutine; debounced out-of-band report on a known target's attach/disconnect transition). - proxmox.Storage: additive parse-only config fields (durable_id sources). - collector StorageObserver seam; Loop.SetTrigger out-of-band report; daemon runs the watchdog as a third goroutine; StorageConfig knobs. - cross-repo golden kept byte-identical with felhom.eu/hub; bidirectional key-set test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 09:59:05 +02:00
parent 1af21a6cac
commit 27b68f043b
22 changed files with 2129 additions and 103 deletions
@@ -0,0 +1,304 @@
+package storage
+
+import (
+	"context"
+	"log/slog"
+	"net"
+	"sync"
+	"time"
+)
+
+// Default watchdog timings (configurable via WatchdogOptions). The poll is FAST (seconds)
+// so a USB drop is caught in seconds, not at the slow ~15-minute host-report cycle; the
+// debounce keeps a flapping drive from storming the hub.
+const (
+	DefaultWatchdogInterval = 8 * time.Second
+	DefaultWatchdogDebounce = 30 * time.Second
+)
+
+// KnownTarget is the watchdog's lightweight view of a target it watches. "Known" means a
+// defined Proxmox storage (and/or a previously-observed-attached one); the watchdog only
+// flags transitions for targets it has seen — it never reports a never-attached device.
+type KnownTarget struct {
+	Name          string
+	Type          string
+	DurableID     string
+	Network       bool   // nfs/cifs/pbs — liveness is a reachability dial, not a device check
+	MountBacked   bool   // usb/local-dir — a drop = its mountpoint disappears
+	BackingDevice string // resolved block device (local targets)
+	MountPath     string // the mountpoint a mount-backed target must occupy
+	ReachEndpoint string // host:port to dial for a network target's reachability
+}
+
+// KnownTargets enumerates the currently-known target set. Production wraps the Observer in
+// CachingKnownTargets so the fast poll doesn't hammer the Proxmox API.
+type KnownTargets interface {
+	Known(ctx context.Context) ([]KnownTarget, error)
+}
+
+// TargetLiveness reports whether one known target is presently up. Production is
+// HostLiveness (device/mount presence + a reachability dial, all non-privileged); tests
+// inject a fake.
+type TargetLiveness interface {
+	Present(ctx context.Context, t KnownTarget) bool
+}
+
+// Transition is one observed state change for a known target (for logging/diagnostics).
+type Transition struct {
+	Name string
+	From string // attached | disconnected
+	To   string
+}
+
+// Watchdog is the third daemon goroutine (alongside the hub loop + reconcile engine). It
+// fast-polls the known target set, detects attached↔disconnected transitions, and triggers
+// an immediate, debounced out-of-band host-report so the hub learns of a drop in seconds.
+//
+// It NEVER mutates anything (Phase A is read-only) — the benign re-mount-by-UUID response
+// to a return lands in Phase B. Here it only observes and signals.
+type Watchdog struct {
+	targets  KnownTargets
+	liveness TargetLiveness
+	interval time.Duration
+	debounce time.Duration
+	trigger  func() // request an out-of-band report (debounced by the watchdog)
+	logger   *slog.Logger
+	now      func() time.Time
+
+	mu       sync.Mutex
+	last     map[string]bool // name -> last observed present (only for seen targets)
+	lastFire time.Time
+	fired    bool // lastFire is valid
+	pending  bool // a transition is awaiting the debounce window
+}
+
+// WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
+// rest default.
+type WatchdogOptions struct {
+	Targets  KnownTargets
+	Liveness TargetLiveness
+	Trigger  func()
+	Interval time.Duration
+	Debounce time.Duration
+	Logger   *slog.Logger
+}
+
+// NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
+// state, just signals nothing) so it degrades cleanly when no report sink is wired.
+func NewWatchdog(opts WatchdogOptions) *Watchdog {
+	interval := opts.Interval
+	if interval <= 0 {
+		interval = DefaultWatchdogInterval
+	}
+	debounce := opts.Debounce
+	if debounce <= 0 {
+		debounce = DefaultWatchdogDebounce
+	}
+	logger := opts.Logger
+	if logger == nil {
+		logger = slog.Default()
+	}
+	trigger := opts.Trigger
+	if trigger == nil {
+		trigger = func() {}
+	}
+	return &Watchdog{
+		targets:  opts.Targets,
+		liveness: opts.Liveness,
+		interval: interval,
+		debounce: debounce,
+		trigger:  trigger,
+		logger:   logger,
+		now:      func() time.Time { return time.Now().UTC() },
+		last:     map[string]bool{},
+	}
+}
+
+// Run fast-polls until ctx is cancelled. The first tick establishes the baseline (no
+// trigger); subsequent ticks detect transitions. Returns nil on ctx cancellation.
+func (w *Watchdog) Run(ctx context.Context) error {
+	if w.targets == nil || w.liveness == nil {
+		w.logger.Info("storage: watchdog idle (no target source / liveness probe configured)")
+		<-ctx.Done()
+		return nil
+	}
+	w.logger.Info("storage: watchdog starting", "interval", w.interval, "debounce", w.debounce)
+	t := time.NewTicker(w.interval)
+	defer t.Stop()
+	w.tick(ctx) // immediate baseline
+	for {
+		select {
+		case <-ctx.Done():
+			w.logger.Info("storage: watchdog shutting down", "reason", ctx.Err())
+			return nil
+		case <-t.C:
+			w.tick(ctx)
+		}
+	}
+}
+
+// tick performs one poll: read the known set, probe each target's liveness, diff against
+// the last-seen state, and fire a debounced trigger on any transition for a SEEN target.
+// It is deterministic given w.now — tests drive it directly with a fake clock.
+func (w *Watchdog) tick(ctx context.Context) {
+	known, err := w.targets.Known(ctx)
+	if err != nil {
+		w.logger.Warn("storage: watchdog could not read known targets; skipping tick", "err", err)
+		return
+	}
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	var transitions []Transition
+	current := make(map[string]bool, len(known))
+	for _, k := range known {
+		present := w.liveness.Present(ctx, k)
+		current[k.Name] = present
+		prev, seen := w.last[k.Name]
+		if !seen {
+			continue // first observation → baseline only (never flag a never-attached drop)
+		}
+		if prev != present {
+			transitions = append(transitions, Transition{Name: k.Name, From: stateStr(prev), To: stateStr(present)})
+		}
+	}
+	// Replace the baseline with the current snapshot (targets no longer known drop out).
+	w.last = current
+
+	now := w.now()
+	if len(transitions) > 0 {
+		for _, tr := range transitions {
+			w.logger.Warn("storage: watchdog detected target state change",
+				"target", tr.Name, "from", tr.From, "to", tr.To)
+		}
+		if !w.fired || now.Sub(w.lastFire) >= w.debounce {
+			w.fire(now, len(transitions))
+		} else {
+			w.pending = true
+			w.logger.Debug("storage: watchdog debouncing transition", "pending_until", w.lastFire.Add(w.debounce))
+		}
+		return
+	}
+	// No new transition, but a debounced one is pending and the window has elapsed → fire.
+	if w.pending && now.Sub(w.lastFire) >= w.debounce {
+		w.fire(now, 0)
+	}
+}
+
+// fire requests the out-of-band report and resets the debounce window. Called under w.mu.
+func (w *Watchdog) fire(now time.Time, n int) {
+	w.lastFire = now
+	w.fired = true
+	w.pending = false
+	w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", n)
+	w.trigger()
+}
+
+func stateStr(present bool) string {
+	if present {
+		return "attached"
+	}
+	return "disconnected"
+}
+
+// --- production liveness + a caching known-target source ---
+
+// HostLiveness is the production TargetLiveness: device + mount presence for local
+// targets (the fast USB-drop signal) and a short reachability dial for network targets.
+// All non-privileged.
+type HostLiveness struct {
+	host        HostReader
+	dialTimeout time.Duration
+	dial        func(network, address string, timeout time.Duration) (net.Conn, error)
+}
+
+// NewHostLiveness builds a HostLiveness over a HostReader. dialTimeout defaults to 3s.
+func NewHostLiveness(host HostReader, dialTimeout time.Duration) *HostLiveness {
+	if host == nil {
+		host = NewProcHostReader()
+	}
+	if dialTimeout <= 0 {
+		dialTimeout = 3 * time.Second
+	}
+	return &HostLiveness{host: host, dialTimeout: dialTimeout, dial: net.DialTimeout}
+}
+
+// Present probes one target without touching Proxmox.
+func (h *HostLiveness) Present(ctx context.Context, t KnownTarget) bool {
+	if t.Network {
+		if t.ReachEndpoint == "" {
+			return true // can't probe → don't false-alarm; the 15-min cycle uses the active flag
+		}
+		conn, err := h.dial("tcp", t.ReachEndpoint, h.dialTimeout)
+		if err != nil {
+			return false
+		}
+		_ = conn.Close()
+		return true
+	}
+	if t.MountBacked {
+		// A mount-backed target (USB / extra disk) is present iff its mountpoint is an
+		// active mount AND the backing device node exists.
+		if !h.mounted(t.MountPath) {
+			return false
+		}
+		return t.BackingDevice == "" || h.host.DeviceExists(t.BackingDevice)
+	}
+	// Non-removable builtin targets (local/lvmthin): treated as present here — they don't
+	// "drop" without the whole host going down, which the heartbeat covers.
+	return true
+}
+
+func (h *HostLiveness) mounted(path string) bool {
+	if path == "" {
+		return false
+	}
+	mounts, err := h.host.Mounts()
+	if err != nil {
+		return false
+	}
+	_, _, ok := exactMountDevice(mounts, path)
+	return ok
+}
+
+// CachingKnownTargets wraps a slow KnownTargets source (the Observer, which hits Proxmox)
+// with a TTL so the fast watchdog poll re-derives the known SET only every ttl, while
+// still probing liveness every tick. A read error returns the last good set (so a
+// transient Proxmox blip doesn't blank the watchdog's world).
+type CachingKnownTargets struct {
+	src    KnownTargets
+	ttl    time.Duration
+	now    func() time.Time
+	mu     sync.Mutex
+	cached []KnownTarget
+	at     time.Time
+	loaded bool
+}
+
+// NewCachingKnownTargets wraps src, refreshing at most every ttl (default 60s).
+func NewCachingKnownTargets(src KnownTargets, ttl time.Duration) *CachingKnownTargets {
+	if ttl <= 0 {
+		ttl = 60 * time.Second
+	}
+	return &CachingKnownTargets{src: src, ttl: ttl, now: func() time.Time { return time.Now().UTC() }}
+}
+
+// Known returns the cached set, refreshing it when the TTL has elapsed.
+func (c *CachingKnownTargets) Known(ctx context.Context) ([]KnownTarget, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := c.now()
+	if c.loaded && now.Sub(c.at) < c.ttl {
+		return c.cached, nil
+	}
+	fresh, err := c.src.Known(ctx)
+	if err != nil {
+		if c.loaded {
+			return c.cached, nil // serve stale rather than blank on a transient error
+		}
+		return nil, err
+	}
+	c.cached, c.at, c.loaded = fresh, now, true
+	return c.cached, nil
+}