v0.5.0: slice 5 Phase B — the host-root surface (mounts + SMART + grow + destructive gate)
The privileged write surface, isolated behind a narrow, arg-validated, adversarially- tested seam (HostOps), the same discipline as the slice-4 gate. Completes slice 5. - internal/storage: HostOps seam + SudoHostOps (systemd .mount units by fs-UUID, detach, SMART, lvs) via sudoers allowlist + fixed arg vectors, no shell; NoopHostOps fallback. - validate.go: strict UUID/mount-path/device/LVM validators + in-process systemd-escape. Headline test: adversarial matrix (metacharacters/traversal/malformed) refused with zero exec. - smart.go: smartctl SATA + NVMe parse, UNKNOWN-degrade; lvs thin-pool metadata fill. - observer enrichment (Observe only): fills smart + thin-pool metadata. - watchdog: benign re-mount response off the poll path (DevicePresent probe, rate-limited). - reconcile: ActionResize (benign, grow-only) + proxmox.ResizeLXC; destructive storage ops (ClassStorageWipe/Decommission) through the slice-4 gate, target-scoped; built+tested, inert live. - --selftest=storage [-watch] live harness; configs/felhom-agent.sudoers; privileged.* knobs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+132
-69
@@ -23,6 +23,7 @@ type KnownTarget struct {
|
||||
Name string
|
||||
Type string
|
||||
DurableID string
|
||||
UUID string // fs-UUID (mount-backed targets) — the by-UUID re-mount key
|
||||
Network bool // nfs/cifs/pbs — liveness is a reachability dial, not a device check
|
||||
MountBacked bool // usb/local-dir — a drop = its mountpoint disappears
|
||||
BackingDevice string // resolved block device (local targets)
|
||||
@@ -36,11 +37,23 @@ type KnownTargets interface {
|
||||
Known(ctx context.Context) ([]KnownTarget, error)
|
||||
}
|
||||
|
||||
// TargetLiveness reports whether one known target is presently up. Production is
|
||||
// HostLiveness (device/mount presence + a reachability dial, all non-privileged); tests
|
||||
// inject a fake.
|
||||
// TargetLiveness reports a known target's liveness. Production is HostLiveness (device/mount
|
||||
// presence + a reachability dial, all non-privileged); tests inject a fake.
|
||||
type TargetLiveness interface {
|
||||
// Present is the "in service" signal: mounted + reachable.
|
||||
Present(ctx context.Context, t KnownTarget) bool
|
||||
// DevicePresent is the "backing device is physically back" signal, independent of
|
||||
// whether it is mounted — the trigger for a benign re-mount of a returned drive.
|
||||
DevicePresent(ctx context.Context, t KnownTarget) bool
|
||||
}
|
||||
|
||||
// Remounter performs the benign re-mount response when a known mount-backed target's device
|
||||
// returns but its mountpoint is missing. The watchdog dispatches to it OFF its poll path
|
||||
// (a goroutine), never synchronously under the lock. Production routes through the gate
|
||||
// (benign) then HostOps.EnsureMount; wired in main.go so storage stays decoupled from
|
||||
// reconcile. A nil Remounter disables the response (observe-only, Phase-A behaviour).
|
||||
type Remounter interface {
|
||||
Remount(ctx context.Context, t KnownTarget)
|
||||
}
|
||||
|
||||
// Transition is one observed state change for a known target (for logging/diagnostics).
|
||||
@@ -57,30 +70,34 @@ type Transition struct {
|
||||
// It NEVER mutates anything (Phase A is read-only) — the benign re-mount-by-UUID response
|
||||
// to a return lands in Phase B. Here it only observes and signals.
|
||||
type Watchdog struct {
|
||||
targets KnownTargets
|
||||
liveness TargetLiveness
|
||||
interval time.Duration
|
||||
debounce time.Duration
|
||||
trigger func() // request an out-of-band report (debounced by the watchdog)
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
targets KnownTargets
|
||||
liveness TargetLiveness
|
||||
remounter Remounter // may be nil (observe-only)
|
||||
interval time.Duration
|
||||
debounce time.Duration
|
||||
trigger func() // request an out-of-band report (debounced by the watchdog)
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
spawn func(func()) // spawn a background task (overridable in tests; default `go f()`)
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
lastRemount map[string]time.Time // name -> last re-mount dispatch (rate-limit)
|
||||
}
|
||||
|
||||
// WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
|
||||
// rest default.
|
||||
// rest default. Remounter is optional (nil = observe-only).
|
||||
type WatchdogOptions struct {
|
||||
Targets KnownTargets
|
||||
Liveness TargetLiveness
|
||||
Trigger func()
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
Targets KnownTargets
|
||||
Liveness TargetLiveness
|
||||
Remounter Remounter
|
||||
Trigger func()
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
|
||||
@@ -103,14 +120,17 @@ func NewWatchdog(opts WatchdogOptions) *Watchdog {
|
||||
trigger = func() {}
|
||||
}
|
||||
return &Watchdog{
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
last: map[string]bool{},
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
remounter: opts.Remounter,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
spawn: func(f func()) { go f() },
|
||||
last: map[string]bool{},
|
||||
lastRemount: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,9 +157,11 @@ func (w *Watchdog) Run(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// tick performs one poll: read the known set, probe each target's liveness, diff against
|
||||
// the last-seen state, and fire a debounced trigger on any transition for a SEEN target.
|
||||
// It is deterministic given w.now — tests drive it directly with a fake clock.
|
||||
// tick performs one poll. Structure: probe liveness OUTSIDE the lock (the probes do IO —
|
||||
// mount reads, dials), then take the lock only for the state diff + debounce decision, then
|
||||
// perform side-effects (report trigger, re-mount dispatch) AFTER unlocking. The re-mount is
|
||||
// handed to a background task — never run synchronously under the lock or on the poll path.
|
||||
// Deterministic given w.now — tests drive it directly with a fake clock.
|
||||
func (w *Watchdog) tick(ctx context.Context) {
|
||||
known, err := w.targets.Known(ctx)
|
||||
if err != nil {
|
||||
@@ -147,52 +169,78 @@ func (w *Watchdog) tick(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
var transitions []Transition
|
||||
current := make(map[string]bool, len(known))
|
||||
for _, k := range known {
|
||||
present := w.liveness.Present(ctx, k)
|
||||
current[k.Name] = present
|
||||
prev, seen := w.last[k.Name]
|
||||
if !seen {
|
||||
continue // first observation → baseline only (never flag a never-attached drop)
|
||||
}
|
||||
if prev != present {
|
||||
transitions = append(transitions, Transition{Name: k.Name, From: stateStr(prev), To: stateStr(present)})
|
||||
}
|
||||
// Probe outside the lock.
|
||||
type probe struct {
|
||||
t KnownTarget
|
||||
present bool
|
||||
devicePresent bool
|
||||
}
|
||||
probes := make([]probe, 0, len(known))
|
||||
for _, k := range known {
|
||||
p := probe{t: k, present: w.liveness.Present(ctx, k)}
|
||||
if k.MountBacked && !p.present {
|
||||
p.devicePresent = w.liveness.DevicePresent(ctx, k)
|
||||
}
|
||||
probes = append(probes, p)
|
||||
}
|
||||
// Replace the baseline with the current snapshot (targets no longer known drop out).
|
||||
w.last = current
|
||||
|
||||
now := w.now()
|
||||
if len(transitions) > 0 {
|
||||
for _, tr := range transitions {
|
||||
w.logger.Warn("storage: watchdog detected target state change",
|
||||
"target", tr.Name, "from", tr.From, "to", tr.To)
|
||||
|
||||
w.mu.Lock()
|
||||
var transitions []Transition
|
||||
var remounts []KnownTarget
|
||||
current := make(map[string]bool, len(probes))
|
||||
for _, p := range probes {
|
||||
current[p.t.Name] = p.present
|
||||
if prev, seen := w.last[p.t.Name]; seen && prev != p.present {
|
||||
transitions = append(transitions, Transition{Name: p.t.Name, From: stateStr(prev), To: stateStr(p.present)})
|
||||
}
|
||||
// Re-mount candidate: a mount-backed target that is NOT mounted but whose backing
|
||||
// device is physically present (a disconnected→device-back state). Rate-limited per
|
||||
// target to the debounce window so a persistent mount failure can't storm HostOps.
|
||||
if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent {
|
||||
if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.debounce {
|
||||
w.lastRemount[p.t.Name] = now
|
||||
remounts = append(remounts, p.t)
|
||||
}
|
||||
}
|
||||
// Once a target is present again, clear its re-mount rate-limit so a future cycle
|
||||
// re-mounts promptly.
|
||||
if p.present {
|
||||
delete(w.lastRemount, p.t.Name)
|
||||
}
|
||||
}
|
||||
w.last = current // targets no longer known drop out
|
||||
|
||||
doFire := false
|
||||
if len(transitions) > 0 {
|
||||
if !w.fired || now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, len(transitions))
|
||||
doFire = true
|
||||
w.lastFire, w.fired, w.pending = now, true, false
|
||||
} else {
|
||||
w.pending = true
|
||||
w.logger.Debug("storage: watchdog debouncing transition", "pending_until", w.lastFire.Add(w.debounce))
|
||||
}
|
||||
return
|
||||
} else if w.pending && now.Sub(w.lastFire) >= w.debounce {
|
||||
doFire = true
|
||||
w.lastFire, w.pending = now, false
|
||||
}
|
||||
// No new transition, but a debounced one is pending and the window has elapsed → fire.
|
||||
if w.pending && now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, 0)
|
||||
}
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
// fire requests the out-of-band report and resets the debounce window. Called under w.mu.
|
||||
func (w *Watchdog) fire(now time.Time, n int) {
|
||||
w.lastFire = now
|
||||
w.fired = true
|
||||
w.pending = false
|
||||
w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", n)
|
||||
w.trigger()
|
||||
// Side-effects, off the lock.
|
||||
for _, tr := range transitions {
|
||||
w.logger.Warn("storage: watchdog detected target state change",
|
||||
"target", tr.Name, "from", tr.From, "to", tr.To)
|
||||
}
|
||||
if doFire {
|
||||
w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", len(transitions))
|
||||
w.trigger()
|
||||
}
|
||||
for _, t := range remounts {
|
||||
t := t
|
||||
w.logger.Info("storage: watchdog dispatching benign re-mount (device returned)",
|
||||
"target", t.Name, "where", t.MountPath)
|
||||
w.spawn(func() { w.remounter.Remount(ctx, t) })
|
||||
}
|
||||
}
|
||||
|
||||
func stateStr(present bool) string {
|
||||
@@ -250,6 +298,21 @@ func (h *HostLiveness) Present(ctx context.Context, t KnownTarget) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// DevicePresent reports whether the backing device is physically present (regardless of
|
||||
// mount state) — the re-mount trigger. Checks /dev/disk/by-uuid/<UUID> first (the by-UUID
|
||||
// link appears when the drive is plugged), then any known backing-device node.
|
||||
func (h *HostLiveness) DevicePresent(ctx context.Context, t KnownTarget) bool {
|
||||
if !t.MountBacked {
|
||||
return false
|
||||
}
|
||||
if t.UUID != "" {
|
||||
if dev, err := ByUUIDDevicePath(t.UUID); err == nil && h.host.DeviceExists(dev) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return t.BackingDevice != "" && h.host.DeviceExists(t.BackingDevice)
|
||||
}
|
||||
|
||||
func (h *HostLiveness) mounted(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
|
||||
Reference in New Issue
Block a user