v0.5.0: slice 5 Phase B — the host-root surface (mounts + SMART + grow + destructive gate)
The privileged write surface, isolated behind a narrow, arg-validated, adversarially- tested seam (HostOps), the same discipline as the slice-4 gate. Completes slice 5. - internal/storage: HostOps seam + SudoHostOps (systemd .mount units by fs-UUID, detach, SMART, lvs) via sudoers allowlist + fixed arg vectors, no shell; NoopHostOps fallback. - validate.go: strict UUID/mount-path/device/LVM validators + in-process systemd-escape. Headline test: adversarial matrix (metacharacters/traversal/malformed) refused with zero exec. - smart.go: smartctl SATA + NVMe parse, UNKNOWN-degrade; lvs thin-pool metadata fill. - observer enrichment (Observe only): fills smart + thin-pool metadata. - watchdog: benign re-mount response off the poll path (DevicePresent probe, rate-limited). - reconcile: ActionResize (benign, grow-only) + proxmox.ResizeLXC; destructive storage ops (ClassStorageWipe/Decommission) through the slice-4 gate, target-scoped; built+tested, inert live. - --selftest=storage [-watch] live harness; configs/felhom-agent.sudoers; privileged.* knobs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// HostOps is the narrow privileged host surface (slice 5 Phase B) — the ONE place the
|
||||
// agent steps outside its Proxmox API token into OS-root. Production is *SudoHostOps
|
||||
// (shells out via a sudoers allowlist, arg vectors, no shell); tests use a fake, so NO
|
||||
// real root runs in the test suite.
|
||||
//
|
||||
// Every method validates its arguments (validate.go) before constructing a command. The
|
||||
// surface is deliberately tiny: persistent mounts (systemd .mount units keyed by fs-UUID),
|
||||
// detach (stop+disable the unit), SMART, and thin-pool metadata.
|
||||
type HostOps interface {
|
||||
// EnsureMount writes + enables a systemd .mount unit for spec (idempotent: re-applying
|
||||
// an existing target is a no-op). Benign — additive, no signature.
|
||||
EnsureMount(ctx context.Context, spec MountSpec) error
|
||||
// Unmount stops + disables the .mount unit for a mountpoint (detach from service). This
|
||||
// is DESTRUCTIVE (deliberately removing a target) — the caller MUST have routed it
|
||||
// through the gate first; HostOps only performs an already-authorized op.
|
||||
Unmount(ctx context.Context, where string) error
|
||||
// SMART returns a parsed health summary for a raw block device, degrading to
|
||||
// {Health: UNKNOWN} when the device exposes no SMART (e.g. a USB-SATA bridge).
|
||||
SMART(ctx context.Context, device string) (hub.SmartSummary, error)
|
||||
// ThinPoolMetadata returns the lvmthin pool's metadata-used fraction (0..1) via lvs.
|
||||
// ok=false when it cannot be read (the field stays null in the report).
|
||||
ThinPoolMetadata(ctx context.Context, vg, pool string) (fraction float64, ok bool)
|
||||
}
|
||||
|
||||
// MountSpec describes a persistent by-UUID mount.
|
||||
type MountSpec struct {
|
||||
Name string // storage name (unit Description only)
|
||||
UUID string // filesystem UUID — validated; becomes What=/dev/disk/by-uuid/<UUID>
|
||||
Where string // mountpoint — validated; the unit name is derived from it
|
||||
FSType string // optional Type=
|
||||
Options string // optional Options=
|
||||
}
|
||||
|
||||
// Binaries holds the absolute paths of the allow-listed binaries (overridable from config
|
||||
// so the sudoers entries and the agent agree on exact paths).
|
||||
type Binaries struct {
|
||||
Systemctl string
|
||||
Install string
|
||||
Smartctl string
|
||||
Lvs string
|
||||
}
|
||||
|
||||
func (b Binaries) withDefaults() Binaries {
|
||||
if b.Systemctl == "" {
|
||||
b.Systemctl = "/usr/bin/systemctl"
|
||||
}
|
||||
if b.Install == "" {
|
||||
b.Install = "/usr/bin/install"
|
||||
}
|
||||
if b.Smartctl == "" {
|
||||
b.Smartctl = "/usr/sbin/smartctl"
|
||||
}
|
||||
if b.Lvs == "" {
|
||||
b.Lvs = "/usr/sbin/lvs"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// SudoHostOps is the production HostOps: it stages a unit file the agent owns, then uses
|
||||
// the sudoers allowlist (`install` it into the unit dir, `systemctl` to manage it,
|
||||
// `smartctl`/`lvs` to read). The Runner execs with an arg vector (no shell) — see
|
||||
// proxmox.ExecRunner in RunnerSudo mode.
|
||||
type SudoHostOps struct {
|
||||
runner proxmox.Runner
|
||||
bins Binaries
|
||||
unitDir string // where enabled units live (e.g. /etc/systemd/system)
|
||||
stageDir string // agent-owned staging dir for unit files before install
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// SudoHostOpsConfig configures a SudoHostOps.
|
||||
type SudoHostOpsConfig struct {
|
||||
Runner proxmox.Runner
|
||||
Bins Binaries
|
||||
UnitDir string // default /etc/systemd/system
|
||||
StageDir string // default <dataDir>/units; must be agent-writable
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewSudoHostOps builds the production privileged surface.
|
||||
func NewSudoHostOps(cfg SudoHostOpsConfig) *SudoHostOps {
|
||||
unitDir := cfg.UnitDir
|
||||
if unitDir == "" {
|
||||
unitDir = "/etc/systemd/system"
|
||||
}
|
||||
stageDir := cfg.StageDir
|
||||
if stageDir == "" {
|
||||
stageDir = "/var/lib/felhom-agent/units"
|
||||
}
|
||||
logger := cfg.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &SudoHostOps{
|
||||
runner: cfg.Runner,
|
||||
bins: cfg.Bins.withDefaults(),
|
||||
unitDir: unitDir,
|
||||
stageDir: stageDir,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureMount validates, renders, stages, installs and enables the .mount unit.
|
||||
func (h *SudoHostOps) EnsureMount(ctx context.Context, spec MountSpec) error {
|
||||
// VALIDATE FIRST — refuse before constructing any command.
|
||||
if err := ValidateUUID(spec.UUID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ValidateMountPath(spec.Where); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateUnitOpt(spec.FSType); err != nil {
|
||||
return fmt.Errorf("storage: fstype: %w", err)
|
||||
}
|
||||
if err := validateUnitOpt(spec.Options); err != nil {
|
||||
return fmt.Errorf("storage: mount options: %w", err)
|
||||
}
|
||||
unitName, err := UnitNameForMount(spec.Where)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
content := renderMountUnit(spec) // uses the validated fields only
|
||||
|
||||
// Stage the unit file (agent-owned dir; no root needed for this write).
|
||||
if err := os.MkdirAll(h.stageDir, 0o700); err != nil {
|
||||
return fmt.Errorf("storage: staging dir: %w", err)
|
||||
}
|
||||
stagePath := filepath.Join(h.stageDir, unitName)
|
||||
if err := os.WriteFile(stagePath, []byte(content), 0o644); err != nil {
|
||||
return fmt.Errorf("storage: staging unit: %w", err)
|
||||
}
|
||||
|
||||
dest := filepath.Join(h.unitDir, unitName)
|
||||
// install as root (atomic copy with fixed mode/owner) — fixed arg vector.
|
||||
if err := h.run(ctx, h.bins.Install, "-o", "root", "-g", "root", "-m", "0644", "--", stagePath, dest); err != nil {
|
||||
return fmt.Errorf("storage: installing unit %s: %w", unitName, err)
|
||||
}
|
||||
if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
|
||||
return fmt.Errorf("storage: daemon-reload: %w", err)
|
||||
}
|
||||
// enable --now both mounts now and persists across reboot. Idempotent.
|
||||
if err := h.run(ctx, h.bins.Systemctl, "enable", "--now", "--", unitName); err != nil {
|
||||
return fmt.Errorf("storage: enabling mount %s: %w", unitName, err)
|
||||
}
|
||||
h.logger.Info("storage: ensured mount", "name", spec.Name, "where", spec.Where, "unit", unitName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmount stops + disables the unit (detach). The caller is responsible for authorization.
|
||||
func (h *SudoHostOps) Unmount(ctx context.Context, where string) error {
|
||||
unitName, err := UnitNameForMount(where)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.run(ctx, h.bins.Systemctl, "stop", "--", unitName); err != nil {
|
||||
return fmt.Errorf("storage: stopping mount %s: %w", unitName, err)
|
||||
}
|
||||
if err := h.run(ctx, h.bins.Systemctl, "disable", "--", unitName); err != nil {
|
||||
return fmt.Errorf("storage: disabling mount %s: %w", unitName, err)
|
||||
}
|
||||
h.logger.Info("storage: unmounted (detached)", "where", where, "unit", unitName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SMART runs `smartctl -a -j <device>` and parses the JSON.
|
||||
func (h *SudoHostOps) SMART(ctx context.Context, device string) (hub.SmartSummary, error) {
|
||||
if err := ValidateSMARTDevice(device); err != nil {
|
||||
return hub.SmartSummary{Health: hub.SmartUnknown}, err
|
||||
}
|
||||
out, stderr, err := h.runner.Run(ctx, h.bins.Smartctl, "-a", "-j", device)
|
||||
if err != nil && len(out) == 0 {
|
||||
// smartctl uses a nonzero exit bitmask even on success; only treat empty output
|
||||
// as a hard failure. A device with no SMART → degrade to UNKNOWN, not an error.
|
||||
return hub.SmartSummary{Health: hub.SmartUnknown}, fmt.Errorf("storage: smartctl %s: %w: %s", device, err, trim(stderr))
|
||||
}
|
||||
return parseSMART(out), nil
|
||||
}
|
||||
|
||||
// ThinPoolMetadata runs `lvs` for the pool and returns its metadata-used fraction.
|
||||
func (h *SudoHostOps) ThinPoolMetadata(ctx context.Context, vg, pool string) (float64, bool) {
|
||||
if err := ValidateLVMName(vg); err != nil {
|
||||
h.logger.Warn("storage: refusing lvs on invalid vg", "vg", vg, "err", err)
|
||||
return 0, false
|
||||
}
|
||||
if err := ValidateLVMName(pool); err != nil {
|
||||
h.logger.Warn("storage: refusing lvs on invalid pool", "pool", pool, "err", err)
|
||||
return 0, false
|
||||
}
|
||||
// --reportformat json, metadata_percent for the specific LV. lv path "vg/pool".
|
||||
out, stderr, err := h.runner.Run(ctx, h.bins.Lvs, "--reportformat", "json", "--units", "b",
|
||||
"-o", "lv_name,data_percent,metadata_percent", "--", vg+"/"+pool)
|
||||
if err != nil && len(out) == 0 {
|
||||
h.logger.Warn("storage: lvs failed", "vg", vg, "pool", pool, "err", err, "stderr", trim(stderr))
|
||||
return 0, false
|
||||
}
|
||||
return parseThinPoolMetadata(out)
|
||||
}
|
||||
|
||||
// run execs an allow-listed command with a fixed arg vector and wraps a nonzero exit.
|
||||
func (h *SudoHostOps) run(ctx context.Context, name string, args ...string) error {
|
||||
_, stderr, err := h.runner.Run(ctx, name, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, trim(stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateUnitOpt rejects metacharacters / newlines in an optional unit value (FSType /
|
||||
// Options) so a crafted value can't inject extra directives into the unit file. Empty is OK.
|
||||
func validateUnitOpt(v string) error {
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
if strings.ContainsAny(v, "\n\r\x00[]=") {
|
||||
return fmt.Errorf("storage: value %q contains forbidden characters", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func trim(b []byte) string {
|
||||
s := strings.TrimSpace(string(b))
|
||||
if len(s) > 300 {
|
||||
return s[:300] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// NoopHostOps is the safe fallback when the privileged surface is unavailable or declined
|
||||
// (a missing sudoers entry must degrade with a clear warning, not crash — slice notes). It
|
||||
// reports SMART as UNKNOWN, no thin-pool metadata, and errors on any write (so a benign
|
||||
// re-mount logs a clear failure rather than silently "succeeding").
|
||||
type NoopHostOps struct{ Logger *slog.Logger }
|
||||
|
||||
func (n NoopHostOps) EnsureMount(context.Context, MountSpec) error {
|
||||
return fmt.Errorf("storage: privileged HostOps not configured; cannot mount")
|
||||
}
|
||||
func (n NoopHostOps) Unmount(context.Context, string) error {
|
||||
return fmt.Errorf("storage: privileged HostOps not configured; cannot unmount")
|
||||
}
|
||||
func (n NoopHostOps) SMART(context.Context, string) (hub.SmartSummary, error) {
|
||||
return hub.SmartSummary{Health: hub.SmartUnknown}, nil
|
||||
}
|
||||
func (n NoopHostOps) ThinPoolMetadata(context.Context, string, string) (float64, bool) {
|
||||
return 0, false
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// scriptRunner returns fixed stdout per binary name (for SMART/lvs parsing tests) and
|
||||
// records calls.
|
||||
type scriptRunner struct {
|
||||
out map[string][]byte // binary name -> stdout
|
||||
calls [][]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *scriptRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
s.calls = append(s.calls, append([]string{name}, args...))
|
||||
return s.out[name], nil, s.err
|
||||
}
|
||||
|
||||
func testStageDir() string { return filepath.Join(os.TempDir(), "felhom-test-units") }
|
||||
|
||||
func TestHostOps_MountLifecycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stage := t.TempDir()
|
||||
unitDir := t.TempDir()
|
||||
rr := &recordingRunner{}
|
||||
ops := NewSudoHostOps(SudoHostOpsConfig{
|
||||
Runner: rr,
|
||||
Bins: Binaries{Systemctl: "/usr/bin/systemctl", Install: "/usr/bin/install"},
|
||||
UnitDir: unitDir,
|
||||
StageDir: stage,
|
||||
Logger: quietLogger(),
|
||||
})
|
||||
|
||||
// A hyphen-free mountpoint so the systemd-escaped unit filename has no backslash — the
|
||||
// backslash escaping is covered by TestSystemdEscapePath; here we just need a filename
|
||||
// that stages on the test OS (Windows treats '\' as a path separator). Production is Linux.
|
||||
spec := MountSpec{Name: "usb-backup", UUID: "0fc63daf-8483-4772-8e79-3d69d8477de4", Where: "/srv/felhom/bulk", FSType: "ext4"}
|
||||
if err := ops.EnsureMount(ctx, spec); err != nil {
|
||||
t.Fatalf("EnsureMount: %v", err)
|
||||
}
|
||||
|
||||
// Expect: install (stage→unitDir), daemon-reload, enable --now -- <unit>.
|
||||
if len(rr.calls) != 3 {
|
||||
t.Fatalf("expected 3 commands, got %d: %v", len(rr.calls), rr.calls)
|
||||
}
|
||||
if rr.calls[0][0] != "/usr/bin/install" || !contains(rr.calls[0], "0644") {
|
||||
t.Errorf("call[0] not the install: %v", rr.calls[0])
|
||||
}
|
||||
if !contains(rr.calls[1], "daemon-reload") {
|
||||
t.Errorf("call[1] not daemon-reload: %v", rr.calls[1])
|
||||
}
|
||||
if !contains(rr.calls[2], "enable") || !contains(rr.calls[2], "--now") {
|
||||
t.Errorf("call[2] not enable --now: %v", rr.calls[2])
|
||||
}
|
||||
|
||||
// The staged unit file is keyed by UUID and uses the validated mountpoint.
|
||||
unitName, _ := UnitNameForMount(spec.Where)
|
||||
content, err := os.ReadFile(filepath.Join(stage, unitName))
|
||||
if err != nil {
|
||||
t.Fatalf("staged unit not written: %v", err)
|
||||
}
|
||||
cs := string(content)
|
||||
if !strings.Contains(cs, "What=/dev/disk/by-uuid/"+spec.UUID) {
|
||||
t.Errorf("unit missing by-uuid What=: %s", cs)
|
||||
}
|
||||
if !strings.Contains(cs, "Where=/srv/felhom/bulk") || !strings.Contains(cs, "Type=ext4") {
|
||||
t.Errorf("unit missing Where/Type: %s", cs)
|
||||
}
|
||||
if !strings.Contains(cs, "WantedBy=multi-user.target") {
|
||||
t.Errorf("unit not enabled-persistent: %s", cs)
|
||||
}
|
||||
|
||||
// Unmount (detach) = stop + disable.
|
||||
rr.calls = nil
|
||||
if err := ops.Unmount(ctx, spec.Where); err != nil {
|
||||
t.Fatalf("Unmount: %v", err)
|
||||
}
|
||||
if len(rr.calls) != 2 || !contains(rr.calls[0], "stop") || !contains(rr.calls[1], "disable") {
|
||||
t.Fatalf("Unmount should stop+disable: %v", rr.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostOps_SMART_SATA(t *testing.T) {
|
||||
sata := []byte(`{
|
||||
"smart_status": {"passed": true},
|
||||
"temperature": {"current": 38},
|
||||
"power_on_time": {"hours": 12345},
|
||||
"ata_smart_attributes": {"table": [
|
||||
{"id": 5, "name": "Reallocated_Sector_Ct", "raw": {"value": 0}},
|
||||
{"id": 197, "name": "Current_Pending_Sector", "raw": {"value": 2}},
|
||||
{"id": 198, "name": "Offline_Uncorrectable", "raw": {"value": 1}}
|
||||
]}
|
||||
}`)
|
||||
ops := &SudoHostOps{runner: &scriptRunner{out: map[string][]byte{"/usr/sbin/smartctl": sata}}, bins: Binaries{}.withDefaults(), logger: quietLogger()}
|
||||
s, err := ops.SMART(context.Background(), "/dev/sda")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Health != hub.SmartPassed {
|
||||
t.Errorf("health = %q, want PASSED", s.Health)
|
||||
}
|
||||
if got := deref(s.TemperatureC); got != 38 {
|
||||
t.Errorf("temp = %d", got)
|
||||
}
|
||||
if deref(s.ReallocatedSectors) != 0 || deref(s.PendingSectors) != 2 || deref(s.OfflineUncorrectable) != 1 {
|
||||
t.Errorf("SATA counters wrong: %+v", s)
|
||||
}
|
||||
if s.MediaErrors != nil || s.PercentageUsed != nil {
|
||||
t.Errorf("NVMe counters must be nil for a SATA disk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostOps_SMART_NVMe(t *testing.T) {
|
||||
nvme := []byte(`{
|
||||
"smart_status": {"passed": true},
|
||||
"nvme_smart_health_information_log": {
|
||||
"critical_warning": 0,
|
||||
"media_errors": 5,
|
||||
"percentage_used": 7,
|
||||
"temperature": 41
|
||||
}
|
||||
}`)
|
||||
ops := &SudoHostOps{runner: &scriptRunner{out: map[string][]byte{"/usr/sbin/smartctl": nvme}}, bins: Binaries{}.withDefaults(), logger: quietLogger()}
|
||||
s, _ := ops.SMART(context.Background(), "/dev/nvme0n1")
|
||||
if s.Health != hub.SmartPassed {
|
||||
t.Errorf("health = %q", s.Health)
|
||||
}
|
||||
if deref(s.CriticalWarning) != 0 || deref(s.MediaErrors) != 5 || deref(s.PercentageUsed) != 7 {
|
||||
t.Errorf("NVMe counters wrong: %+v", s)
|
||||
}
|
||||
if deref(s.TemperatureC) != 41 {
|
||||
t.Errorf("nvme temp = %v", s.TemperatureC)
|
||||
}
|
||||
if s.ReallocatedSectors != nil {
|
||||
t.Errorf("SATA counters must be nil for an NVMe disk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostOps_SMART_Unsupported(t *testing.T) {
|
||||
// A USB-SATA bridge that exposes no SMART: smartctl returns minimal JSON (no
|
||||
// smart_status) and a nonzero exit. We degrade to UNKNOWN, not an error.
|
||||
ops := &SudoHostOps{
|
||||
runner: &scriptRunner{out: map[string][]byte{"/usr/sbin/smartctl": []byte(`{"device":{"name":"/dev/sdc"}}`)}, err: errExit(2)},
|
||||
bins: Binaries{}.withDefaults(), logger: quietLogger(),
|
||||
}
|
||||
s, err := ops.SMART(context.Background(), "/dev/sdc")
|
||||
if err != nil {
|
||||
t.Fatalf("unsupported SMART must degrade, not error: %v", err)
|
||||
}
|
||||
if s.Health != hub.SmartUnknown {
|
||||
t.Errorf("health = %q, want UNKNOWN", s.Health)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostOps_ThinPoolMetadata(t *testing.T) {
|
||||
lvs := []byte(`{"report":[{"lv":[{"lv_name":"data","data_percent":"42.00","metadata_percent":"10.50"}]}]}`)
|
||||
ops := &SudoHostOps{runner: &scriptRunner{out: map[string][]byte{"/usr/sbin/lvs": lvs}}, bins: Binaries{}.withDefaults(), logger: quietLogger()}
|
||||
frac, ok := ops.ThinPoolMetadata(context.Background(), "pve", "data")
|
||||
if !ok {
|
||||
t.Fatal("expected metadata fraction")
|
||||
}
|
||||
if frac < 0.104 || frac > 0.106 {
|
||||
t.Errorf("metadata fraction = %v, want ~0.105", frac)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(ss []string, want string) bool {
|
||||
for _, s := range ss {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func deref(p *int) int {
|
||||
if p == nil {
|
||||
return -1
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// errExit is a stand-in for a nonzero exit error from the runner.
|
||||
type errExitT int
|
||||
|
||||
func (e errExitT) Error() string { return "exit status nonzero" }
|
||||
func errExit(code int) error { return errExitT(code) }
|
||||
@@ -0,0 +1,44 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// renderMountUnit builds the systemd .mount unit content for a (already-validated) spec.
|
||||
// Keyed by fs-UUID via What=/dev/disk/by-uuid/<UUID> so it survives /dev/sdX renumbering;
|
||||
// WantedBy=multi-user.target so `enable` makes it persist across reboot.
|
||||
//
|
||||
// All interpolated values are pre-validated by the caller (ValidateUUID / ValidateMountPath
|
||||
// / validateUnitOpt), so no value here can carry a newline or inject an extra directive.
|
||||
func renderMountUnit(spec MountSpec) string {
|
||||
what := byUUIDDir + "/" + spec.UUID
|
||||
var b strings.Builder
|
||||
b.WriteString("# Managed by felhom-agent — do not edit by hand.\n")
|
||||
b.WriteString("[Unit]\n")
|
||||
fmt.Fprintf(&b, "Description=Felhom storage mount %s\n", sanitizeDesc(spec.Name))
|
||||
b.WriteString("After=local-fs-pre.target\n")
|
||||
b.WriteString("\n[Mount]\n")
|
||||
fmt.Fprintf(&b, "What=%s\n", what)
|
||||
fmt.Fprintf(&b, "Where=%s\n", spec.Where)
|
||||
if spec.FSType != "" {
|
||||
fmt.Fprintf(&b, "Type=%s\n", spec.FSType)
|
||||
}
|
||||
if spec.Options != "" {
|
||||
fmt.Fprintf(&b, "Options=%s\n", spec.Options)
|
||||
}
|
||||
b.WriteString("\n[Install]\n")
|
||||
b.WriteString("WantedBy=multi-user.target\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sanitizeDesc keeps the Description line single-line and harmless (it is cosmetic; the
|
||||
// name is already a Proxmox storage id, but be defensive against any newline).
|
||||
func sanitizeDesc(name string) string {
|
||||
name = strings.ReplaceAll(name, "\n", " ")
|
||||
name = strings.ReplaceAll(name, "\r", " ")
|
||||
if name == "" {
|
||||
return "(unnamed)"
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
@@ -26,29 +27,37 @@ type StorageAPI interface {
|
||||
}
|
||||
|
||||
// Observer builds the observed storage view from Proxmox + non-privileged host reads.
|
||||
// In Phase B it also (optionally) enriches the reported view with the privileged reads —
|
||||
// SMART + thin-pool metadata — via HostOps; a nil ops keeps the Phase-A behaviour
|
||||
// (SMART UNKNOWN, metadata null).
|
||||
type Observer struct {
|
||||
api StorageAPI
|
||||
host HostReader
|
||||
ops HostOps
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewObserver builds an Observer. host defaults to a ProcHostReader; logger to the
|
||||
// default. A nil api makes Observe/Known return an error (misconfiguration), never panic.
|
||||
func NewObserver(api StorageAPI, host HostReader, logger *slog.Logger) *Observer {
|
||||
// default. ops is the privileged surface for SMART/lvs — nil disables those (Phase-A
|
||||
// behaviour). A nil api makes Observe/Known return an error (misconfiguration), never panic.
|
||||
func NewObserver(api StorageAPI, host HostReader, ops HostOps, logger *slog.Logger) *Observer {
|
||||
if host == nil {
|
||||
host = NewProcHostReader()
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Observer{api: api, host: host, logger: logger}
|
||||
return &Observer{api: api, host: host, ops: ops, logger: logger}
|
||||
}
|
||||
|
||||
// observed is the rich internal view of one target, from which both the reported
|
||||
// hub.StorageTarget and the watchdog's KnownTarget are projected.
|
||||
// hub.StorageTarget and the watchdog's KnownTarget are projected. src/cat are kept for
|
||||
// Observe-time privileged enrichment (NOT used by the watchdog's Known path).
|
||||
type observed struct {
|
||||
target hub.StorageTarget
|
||||
known KnownTarget
|
||||
src proxmox.Storage
|
||||
cat storageCategory
|
||||
}
|
||||
|
||||
// Observe builds the reported []hub.StorageTarget. A non-nil error means the Proxmox read
|
||||
@@ -61,11 +70,43 @@ func (o *Observer) Observe(ctx context.Context) ([]hub.StorageTarget, error) {
|
||||
}
|
||||
out := make([]hub.StorageTarget, 0, len(snap))
|
||||
for _, s := range snap {
|
||||
out = append(out, s.target)
|
||||
out = append(out, o.enrich(ctx, s))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// enrich adds the PRIVILEGED reads (SMART, thin-pool metadata) on top of the base target.
|
||||
// Only Observe calls this (the watchdog's Known path skips it — these are the slow,
|
||||
// root-shelling reads). A nil ops or a per-target failure degrades gracefully: SMART stays
|
||||
// UNKNOWN, metadata stays null.
|
||||
func (o *Observer) enrich(ctx context.Context, ob observed) hub.StorageTarget {
|
||||
t := ob.target
|
||||
if o.ops == nil {
|
||||
return t
|
||||
}
|
||||
// SMART: only for dir-backed targets with a resolvable whole-disk device.
|
||||
if ob.cat == catDir && t.BackingDevice != "" {
|
||||
if dev, ok := smartDeviceFor(t.BackingDevice); ok {
|
||||
if sm, err := o.ops.SMART(ctx, dev); err != nil {
|
||||
o.logger.Warn("storage: SMART read failed; health UNKNOWN", "device", dev, "err", err)
|
||||
} else {
|
||||
t.Smart = sm
|
||||
}
|
||||
}
|
||||
}
|
||||
// Thin-pool metadata fill (the value Phase A left null): lvs on the vg/pool.
|
||||
if t.Type == hub.StorageTypeLVMThin && t.ThinPool != nil && ob.src.VGName != "" && ob.src.ThinPool != "" {
|
||||
if frac, ok := o.ops.ThinPoolMetadata(ctx, ob.src.VGName, ob.src.ThinPool); ok {
|
||||
t.ThinPool.MetadataUsedFraction = &frac
|
||||
if frac >= thinPoolWarnFraction {
|
||||
o.logger.Warn("storage: lvmthin pool METADATA fill is high (exhaustion corrupts the pool like data exhaustion)",
|
||||
"storage", t.Name, "metadata_used_fraction", frac)
|
||||
}
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Known projects the snapshot to the watchdog's lightweight KnownTarget set. Same Proxmox
|
||||
// + host reads as Observe — callers that poll it fast should wrap it in a cache (the
|
||||
// watchdog uses CachingKnownTargets).
|
||||
@@ -194,10 +235,13 @@ func (o *Observer) build(s proxmox.Storage, mounts []Mount) observed {
|
||||
|
||||
return observed{
|
||||
target: tgt,
|
||||
src: s,
|
||||
cat: category,
|
||||
known: KnownTarget{
|
||||
Name: s.Storage,
|
||||
Type: typ,
|
||||
DurableID: durableID,
|
||||
UUID: uuid,
|
||||
Network: category == catNetwork,
|
||||
MountBacked: typ == hub.StorageTypeUSB || typ == hub.StorageTypeLocalDir,
|
||||
BackingDevice: backingDevice,
|
||||
@@ -207,6 +251,27 @@ func (o *Observer) build(s proxmox.Storage, mounts []Mount) observed {
|
||||
}
|
||||
}
|
||||
|
||||
// smartDeviceFor maps a backing device (possibly a partition) to its whole-disk path for
|
||||
// smartctl (which targets the disk, not the partition). Returns ok=false when the result
|
||||
// isn't a recognized raw disk (e.g. device-mapper / LVM), so SMART is simply skipped.
|
||||
func smartDeviceFor(device string) (string, bool) {
|
||||
dev := device
|
||||
if m := reNVMePart.FindStringSubmatch(device); m != nil {
|
||||
dev = m[1] // /dev/nvme0n1p2 -> /dev/nvme0n1
|
||||
} else if m := reSDPart.FindStringSubmatch(device); m != nil {
|
||||
dev = m[1] // /dev/sdb1 -> /dev/sdb
|
||||
}
|
||||
if ValidateSMARTDevice(dev) != nil {
|
||||
return "", false
|
||||
}
|
||||
return dev, true
|
||||
}
|
||||
|
||||
var (
|
||||
reNVMePart = regexp.MustCompile(`^(/dev/nvme[0-9]+n[0-9]+)p[0-9]+$`)
|
||||
reSDPart = regexp.MustCompile(`^(/dev/(?:sd|hd|vd)[a-z]+)[0-9]+$`)
|
||||
)
|
||||
|
||||
// reachable decides whether the target is currently usable.
|
||||
// - usb / local-dir: a Felhom extra/removable dir storage is realized as its OWN
|
||||
// mountpoint, so reachable = it is currently an exact mount AND its device node exists.
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestObserve_BuildsTargetsFromProxmoxAndHostReads(t *testing.T) {
|
||||
removable: map[string]bool{"/dev/sdb1": true, "/dev/mapper/pve-root": false},
|
||||
}
|
||||
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
got, err := NewObserver(api, host, nil, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Observe: %v", err)
|
||||
}
|
||||
@@ -156,6 +156,72 @@ func TestObserve_BuildsTargetsFromProxmoxAndHostReads(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHostOps fills SMART + thin-pool metadata for the enrichment test.
|
||||
type fakeHostOps struct {
|
||||
smartByDevice map[string]hub.SmartSummary
|
||||
metaByPool map[string]float64 // "vg/pool" -> fraction
|
||||
smartDevices []string // records which devices SMART was called on
|
||||
}
|
||||
|
||||
func (f *fakeHostOps) EnsureMount(context.Context, MountSpec) error { return nil }
|
||||
func (f *fakeHostOps) Unmount(context.Context, string) error { return nil }
|
||||
func (f *fakeHostOps) SMART(_ context.Context, device string) (hub.SmartSummary, error) {
|
||||
f.smartDevices = append(f.smartDevices, device)
|
||||
if s, ok := f.smartByDevice[device]; ok {
|
||||
return s, nil
|
||||
}
|
||||
return hub.SmartSummary{Health: hub.SmartUnknown}, nil
|
||||
}
|
||||
func (f *fakeHostOps) ThinPoolMetadata(_ context.Context, vg, pool string) (float64, bool) {
|
||||
v, ok := f.metaByPool[vg+"/"+pool]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func TestObserve_EnrichesSMARTAndThinPoolMetadata(t *testing.T) {
|
||||
api := &fakeStorageAPI{
|
||||
node: "demo-felhom",
|
||||
cluster: []proxmox.Storage{
|
||||
{Storage: "local-lvm", Type: "lvmthin", VGName: "pve", ThinPool: "data"},
|
||||
{Storage: "usb-backup", Type: "dir", Path: "/mnt/usb-backup"},
|
||||
},
|
||||
nodeSt: []proxmox.Storage{
|
||||
{Storage: "local-lvm", Type: "lvmthin", Active: 1, UsedFraction: 0.4},
|
||||
{Storage: "usb-backup", Type: "dir", Path: "/mnt/usb-backup", Active: 1},
|
||||
},
|
||||
}
|
||||
host := &fakeHostReader{
|
||||
mounts: []Mount{{Device: "/dev/sdb1", MountPoint: "/mnt/usb-backup", FSType: "ext4"}},
|
||||
uuids: map[string]string{"/dev/sdb1": "1111-2222"},
|
||||
exists: map[string]bool{"/dev/sdb1": true},
|
||||
removable: map[string]bool{"/dev/sdb1": true},
|
||||
}
|
||||
ops := &fakeHostOps{
|
||||
smartByDevice: map[string]hub.SmartSummary{"/dev/sdb": {Health: hub.SmartPassed}},
|
||||
metaByPool: map[string]float64{"pve/data": 0.12},
|
||||
}
|
||||
got, err := NewObserver(api, host, ops, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := byName(got)
|
||||
|
||||
// SMART runs on the WHOLE disk (/dev/sdb), not the partition (/dev/sdb1).
|
||||
if len(ops.smartDevices) != 1 || ops.smartDevices[0] != "/dev/sdb" {
|
||||
t.Errorf("SMART should target the whole disk /dev/sdb, got %v", ops.smartDevices)
|
||||
}
|
||||
if m["usb-backup"].Smart.Health != hub.SmartPassed {
|
||||
t.Errorf("usb SMART not enriched: %+v", m["usb-backup"].Smart)
|
||||
}
|
||||
// lvmthin metadata fill (Phase B) is now populated.
|
||||
lvm := m["local-lvm"]
|
||||
if lvm.ThinPool == nil || lvm.ThinPool.MetadataUsedFraction == nil {
|
||||
t.Fatalf("lvmthin metadata fill not enriched: %+v", lvm.ThinPool)
|
||||
}
|
||||
if *lvm.ThinPool.MetadataUsedFraction != 0.12 {
|
||||
t.Errorf("metadata fraction = %v, want 0.12", *lvm.ThinPool.MetadataUsedFraction)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserve_USBUnpluggedIsDisconnected(t *testing.T) {
|
||||
api := &fakeStorageAPI{
|
||||
node: "demo-felhom",
|
||||
@@ -170,7 +236,7 @@ func TestObserve_USBUnpluggedIsDisconnected(t *testing.T) {
|
||||
host := &fakeHostReader{
|
||||
mounts: []Mount{{Device: "/dev/mapper/pve-root", MountPoint: "/", FSType: "ext4"}}, // no /mnt/usb-backup
|
||||
}
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
got, err := NewObserver(api, host, nil, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -187,7 +253,7 @@ func TestObserve_USBUnpluggedIsDisconnected(t *testing.T) {
|
||||
|
||||
func TestObserve_ProxmoxErrorIsFatalForStorage(t *testing.T) {
|
||||
api := &fakeStorageAPI{node: "n", listErr: context.DeadlineExceeded}
|
||||
if _, err := NewObserver(api, &fakeHostReader{}, quietLogger()).Observe(context.Background()); err == nil {
|
||||
if _, err := NewObserver(api, &fakeHostReader{}, nil, quietLogger()).Observe(context.Background()); err == nil {
|
||||
t.Fatal("a Proxmox read error must surface (the collector then omits storage this cycle)")
|
||||
}
|
||||
}
|
||||
@@ -199,7 +265,7 @@ func TestObserve_MountReadFailureDegradesNotFatal(t *testing.T) {
|
||||
nodeSt: []proxmox.Storage{{Storage: "local-lvm", Type: "lvmthin", Active: 1, UsedFraction: 0.1}},
|
||||
}
|
||||
host := &fakeHostReader{mountsErr: io.ErrUnexpectedEOF}
|
||||
got, err := NewObserver(api, host, quietLogger()).Observe(context.Background())
|
||||
got, err := NewObserver(api, host, nil, quietLogger()).Observe(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("a host mount-read failure must degrade, not fail: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// smartctlJSON is the lenient subset of `smartctl -a -j` output we read. Pointers detect
|
||||
// presence so an absent section (e.g. NVMe fields on a SATA disk, or no SMART at all on a
|
||||
// USB bridge) decodes cleanly to nil and we degrade to UNKNOWN.
|
||||
type smartctlJSON struct {
|
||||
SmartStatus *struct {
|
||||
Passed bool `json:"passed"`
|
||||
} `json:"smart_status"`
|
||||
Temperature *struct {
|
||||
Current *int `json:"current"`
|
||||
} `json:"temperature"`
|
||||
PowerOnTime *struct {
|
||||
Hours *int `json:"hours"`
|
||||
} `json:"power_on_time"`
|
||||
// SATA/ATA attribute table.
|
||||
ATA *struct {
|
||||
Table []struct {
|
||||
ID int `json:"id"`
|
||||
Raw struct {
|
||||
Value int64 `json:"value"`
|
||||
} `json:"raw"`
|
||||
} `json:"table"`
|
||||
} `json:"ata_smart_attributes"`
|
||||
// NVMe health log.
|
||||
NVMe *struct {
|
||||
CriticalWarning *int `json:"critical_warning"`
|
||||
MediaErrors *int64 `json:"media_errors"`
|
||||
PercentageUsed *int `json:"percentage_used"`
|
||||
Temperature *int `json:"temperature"`
|
||||
} `json:"nvme_smart_health_information_log"`
|
||||
}
|
||||
|
||||
// SATA attribute IDs we surface.
|
||||
const (
|
||||
ataReallocatedSectorCt = 5
|
||||
ataCurrentPending = 197
|
||||
ataOfflineUncorrect = 198
|
||||
)
|
||||
|
||||
// parseSMART maps smartctl JSON to a hub.SmartSummary, handling SATA + NVMe and degrading
|
||||
// to UNKNOWN when health is not reported. A device populates only its own attribute set.
|
||||
func parseSMART(raw []byte) hub.SmartSummary {
|
||||
s := hub.SmartSummary{Health: hub.SmartUnknown}
|
||||
if len(raw) == 0 {
|
||||
return s
|
||||
}
|
||||
var j smartctlJSON
|
||||
if err := json.Unmarshal(raw, &j); err != nil {
|
||||
return s // unparseable → UNKNOWN (never an error to the report)
|
||||
}
|
||||
|
||||
if j.SmartStatus != nil {
|
||||
if j.SmartStatus.Passed {
|
||||
s.Health = hub.SmartPassed
|
||||
} else {
|
||||
s.Health = hub.SmartFailing
|
||||
}
|
||||
}
|
||||
if j.Temperature != nil && j.Temperature.Current != nil {
|
||||
s.TemperatureC = j.Temperature.Current
|
||||
}
|
||||
if j.PowerOnTime != nil && j.PowerOnTime.Hours != nil {
|
||||
s.PowerOnHours = j.PowerOnTime.Hours
|
||||
}
|
||||
|
||||
// SATA attributes.
|
||||
if j.ATA != nil {
|
||||
for _, a := range j.ATA.Table {
|
||||
switch a.ID {
|
||||
case ataReallocatedSectorCt:
|
||||
s.ReallocatedSectors = intPtr(int(a.Raw.Value))
|
||||
case ataCurrentPending:
|
||||
s.PendingSectors = intPtr(int(a.Raw.Value))
|
||||
case ataOfflineUncorrect:
|
||||
s.OfflineUncorrectable = intPtr(int(a.Raw.Value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NVMe attributes.
|
||||
if j.NVMe != nil {
|
||||
s.CriticalWarning = j.NVMe.CriticalWarning
|
||||
if j.NVMe.MediaErrors != nil {
|
||||
s.MediaErrors = intPtr(int(*j.NVMe.MediaErrors))
|
||||
}
|
||||
s.PercentageUsed = j.NVMe.PercentageUsed
|
||||
// NVMe reports temperature in its own log when the top-level block is absent.
|
||||
if s.TemperatureC == nil && j.NVMe.Temperature != nil {
|
||||
s.TemperatureC = j.NVMe.Temperature
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// lvsReport is the lenient subset of `lvs --reportformat json` output.
|
||||
type lvsReport struct {
|
||||
Report []struct {
|
||||
LV []struct {
|
||||
LVName string `json:"lv_name"`
|
||||
DataPercent string `json:"data_percent"`
|
||||
MetadataPercent string `json:"metadata_percent"`
|
||||
} `json:"lv"`
|
||||
} `json:"report"`
|
||||
}
|
||||
|
||||
// parseThinPoolMetadata extracts the metadata-used fraction (0..1) from lvs JSON. lvs
|
||||
// reports percentages as decimal strings (e.g. "10.50"); an empty string means "not a thin
|
||||
// pool / not applicable" → ok=false.
|
||||
func parseThinPoolMetadata(raw []byte) (float64, bool) {
|
||||
if len(raw) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
var r lvsReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for _, rep := range r.Report {
|
||||
for _, lv := range rep.LV {
|
||||
if lv.MetadataPercent == "" {
|
||||
continue
|
||||
}
|
||||
pct, err := strconv.ParseFloat(lv.MetadataPercent, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
return pct / 100, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func intPtr(v int) *int { return &v }
|
||||
@@ -0,0 +1,178 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file is the security boundary for the privileged host surface (slice 5 Phase B).
|
||||
// EVERY argument that will reach a root shell-out is validated HERE, before any command
|
||||
// is constructed — the SudoHostOps methods refuse on a validation error and never build an
|
||||
// arg vector, let alone exec. The adversarial matrix in validate_test.go is the proof that
|
||||
// the "aggressive write side" is not a loose one: shell metacharacters, path traversal, and
|
||||
// malformed inputs are rejected up front. Combined with arg-vector exec (never a shell
|
||||
// string), a validated input cannot inject.
|
||||
|
||||
var (
|
||||
// fs-UUIDs: ext/xfs are 8-4-4-4-12 lowercase hex; FAT/vFAT are "XXXX-XXXX" (upper
|
||||
// hex); others vary. Accept hex groups joined by single hyphens, length-bounded.
|
||||
// This rejects '/', '.', whitespace, and every shell metacharacter by construction.
|
||||
reUUID = regexp.MustCompile(`^[A-Fa-f0-9]{4,}(-[A-Fa-f0-9]+){0,4}$`)
|
||||
|
||||
// SMART device: a strict whitelist of real block-disk patterns under /dev. No
|
||||
// /dev/disk/by-* symlinks, no device-mapper, no traversal — just the raw disks
|
||||
// smartctl is run against. Anything else is refused.
|
||||
reSMARTDevice = regexp.MustCompile(`^/dev/(sd[a-z]+|nvme[0-9]+n[0-9]+|hd[a-z]+|vd[a-z]+)$`)
|
||||
|
||||
// LVM VG / pool names: LVM permits [A-Za-z0-9._+-]; we forbid leading '-' (would look
|
||||
// like a flag) and cap the length.
|
||||
reLVMName = regexp.MustCompile(`^[A-Za-z0-9_+.][A-Za-z0-9_+.-]*$`)
|
||||
|
||||
// A single safe path segment (for mountpoint validation). No metacharacters; "." and
|
||||
// ".." are rejected separately as traversal.
|
||||
rePathSegment = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||
)
|
||||
|
||||
const (
|
||||
maxUUIDLen = 40
|
||||
maxPathLen = 255
|
||||
maxLVMLen = 128
|
||||
byUUIDDir = "/dev/disk/by-uuid"
|
||||
maxMountSeg = 32 // a sane cap on mountpoint depth
|
||||
)
|
||||
|
||||
// ValidateUUID accepts a filesystem UUID for use in a by-uuid device path. It is the
|
||||
// load-bearing check (the UUID is the DR re-attach key AND a shell-out argument).
|
||||
func ValidateUUID(uuid string) error {
|
||||
if uuid == "" {
|
||||
return fmt.Errorf("storage: empty UUID")
|
||||
}
|
||||
if len(uuid) > maxUUIDLen {
|
||||
return fmt.Errorf("storage: UUID too long (%d > %d)", len(uuid), maxUUIDLen)
|
||||
}
|
||||
if !reUUID.MatchString(uuid) {
|
||||
return fmt.Errorf("storage: invalid UUID %q (want hex groups, no metacharacters)", uuid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ByUUIDDevicePath returns the validated /dev/disk/by-uuid/<uuid> path for a mount unit's
|
||||
// What=. Device paths for mounting are ALWAYS confined to this directory — we never accept
|
||||
// an arbitrary device path from any source.
|
||||
func ByUUIDDevicePath(uuid string) (string, error) {
|
||||
if err := ValidateUUID(uuid); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return byUUIDDir + "/" + uuid, nil
|
||||
}
|
||||
|
||||
// ValidateMountPath accepts an absolute mountpoint with no traversal and no metacharacters.
|
||||
// Each segment must be a safe token; "." / ".." segments are rejected; the bare root "/"
|
||||
// is rejected (we never manage a mount at root).
|
||||
func ValidateMountPath(path string) error {
|
||||
if path == "" || path[0] != '/' {
|
||||
return fmt.Errorf("storage: mount path must be absolute, got %q", path)
|
||||
}
|
||||
if len(path) > maxPathLen {
|
||||
return fmt.Errorf("storage: mount path too long (%d > %d)", len(path), maxPathLen)
|
||||
}
|
||||
if strings.ContainsAny(path, "\x00\n\r\t") {
|
||||
return fmt.Errorf("storage: mount path contains control characters")
|
||||
}
|
||||
segs := nonEmptySegments(path)
|
||||
if len(segs) == 0 {
|
||||
return fmt.Errorf("storage: refusing to manage a mount at %q", path)
|
||||
}
|
||||
if len(segs) > maxMountSeg {
|
||||
return fmt.Errorf("storage: mount path too deep")
|
||||
}
|
||||
for _, s := range segs {
|
||||
if s == "." || s == ".." {
|
||||
return fmt.Errorf("storage: mount path traversal segment %q in %q", s, path)
|
||||
}
|
||||
if !rePathSegment.MatchString(s) {
|
||||
return fmt.Errorf("storage: invalid mount path segment %q in %q", s, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateSMARTDevice accepts only a raw block-disk path (sdX/nvmeXnY/hdX/vdX) under /dev.
|
||||
func ValidateSMARTDevice(device string) error {
|
||||
if !reSMARTDevice.MatchString(device) {
|
||||
return fmt.Errorf("storage: refusing smartctl on non-whitelisted device %q", device)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateLVMName accepts an LVM VG or LV (pool) name.
|
||||
func ValidateLVMName(name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("storage: empty LVM name")
|
||||
}
|
||||
if len(name) > maxLVMLen {
|
||||
return fmt.Errorf("storage: LVM name too long")
|
||||
}
|
||||
if !reLVMName.MatchString(name) {
|
||||
return fmt.Errorf("storage: invalid LVM name %q", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnitNameForMount returns the systemd .mount unit name for a (validated) mountpoint. A
|
||||
// .mount unit's name MUST be the systemd-escaped mountpoint — this is computed
|
||||
// deterministically from the already-validated path, so the result is inherently safe to
|
||||
// pass in an arg vector (no shell).
|
||||
func UnitNameForMount(where string) (string, error) {
|
||||
if err := ValidateMountPath(where); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return systemdEscapePath(where) + ".mount", nil
|
||||
}
|
||||
|
||||
// nonEmptySegments splits a path on '/', dropping empties (so "//a///b/" → [a b]).
|
||||
func nonEmptySegments(path string) []string {
|
||||
parts := strings.Split(path, "/")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// systemdEscapePath replicates `systemd-escape --path`: strip leading/trailing slashes and
|
||||
// collapse internal repeats, then escape each char — '/' → '-', alnum/'_' kept, '.' kept
|
||||
// (except a leading '.'), everything else (including a literal '-') → '\xNN'. The empty
|
||||
// path / "/" escapes to "-". Computed in-process so no `systemd-escape` shell-out / sudoers
|
||||
// entry is needed.
|
||||
func systemdEscapePath(path string) string {
|
||||
segs := nonEmptySegments(path)
|
||||
if len(segs) == 0 {
|
||||
return "-"
|
||||
}
|
||||
joined := strings.Join(segs, "/")
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(joined); i++ {
|
||||
c := joined[i]
|
||||
switch {
|
||||
case c == '/':
|
||||
b.WriteByte('-')
|
||||
case i == 0 && c == '.':
|
||||
b.WriteString(`\x2e`)
|
||||
case isAlnum(c) || c == '_':
|
||||
b.WriteByte(c)
|
||||
case c == '.':
|
||||
b.WriteByte('.')
|
||||
default:
|
||||
fmt.Fprintf(&b, `\x%02x`, c)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func isAlnum(c byte) bool {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// recordingRunner records every command it is asked to run (and never execs anything). The
|
||||
// adversarial matrix asserts that a rejected argument means ZERO commands were constructed —
|
||||
// the validator is the wall, not the exec.
|
||||
type recordingRunner struct {
|
||||
calls [][]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *recordingRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
r.calls = append(r.calls, append([]string{name}, args...))
|
||||
return nil, nil, r.err
|
||||
}
|
||||
|
||||
// --- The headline: the arg-validator adversarial matrix. ---
|
||||
|
||||
func TestValidateUUID_AdversarialMatrix(t *testing.T) {
|
||||
good := []string{
|
||||
"0fc63daf-8483-4772-8e79-3d69d8477de4", // ext4
|
||||
"1234-ABCD", // FAT
|
||||
"deadbeefdeadbeef", // NTFS-ish 16 hex
|
||||
}
|
||||
for _, u := range good {
|
||||
if err := ValidateUUID(u); err != nil {
|
||||
t.Errorf("ValidateUUID(%q) rejected a valid UUID: %v", u, err)
|
||||
}
|
||||
}
|
||||
bad := []string{
|
||||
"", // empty
|
||||
"../../etc/shadow", // traversal
|
||||
"abcd; rm -rf /", // shell metacharacters
|
||||
"abcd$(reboot)", // command substitution
|
||||
"abcd`reboot`", // backticks
|
||||
"abcd&whoami", // &
|
||||
"abcd|cat", // pipe
|
||||
"abcd\nreboot", // newline
|
||||
"abcd /dev/sda", // space + extra arg
|
||||
"g00dlooking-but-z-not-hex", // non-hex
|
||||
"/dev/disk/by-uuid/abcd", // a path, not a uuid
|
||||
strings.Repeat("a", maxUUIDLen+1), // too long
|
||||
"abcd\x00", // NUL
|
||||
}
|
||||
for _, u := range bad {
|
||||
if err := ValidateUUID(u); err == nil {
|
||||
t.Errorf("ValidateUUID(%q) ACCEPTED a hostile UUID", u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMountPath_AdversarialMatrix(t *testing.T) {
|
||||
good := []string{"/mnt/usb-backup", "/srv/felhom/bulk", "/mnt/data_1"}
|
||||
for _, p := range good {
|
||||
if err := ValidateMountPath(p); err != nil {
|
||||
t.Errorf("ValidateMountPath(%q) rejected a valid path: %v", p, err)
|
||||
}
|
||||
}
|
||||
bad := []string{
|
||||
"", // empty
|
||||
"relative/path", // not absolute
|
||||
"/", // bare root
|
||||
"/mnt/../etc", // traversal
|
||||
"/mnt/./x", // dot segment
|
||||
"/mnt/usb backup", // space
|
||||
"/mnt/usb;reboot", // metacharacter
|
||||
"/mnt/$(reboot)", // command substitution
|
||||
"/mnt/x\nWhat=/dev/sda", // newline → unit-file injection attempt
|
||||
"/mnt/x\x00", // NUL
|
||||
"/mnt/x`reboot`", // backticks
|
||||
}
|
||||
for _, p := range bad {
|
||||
if err := ValidateMountPath(p); err == nil {
|
||||
t.Errorf("ValidateMountPath(%q) ACCEPTED a hostile path", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSMARTDevice_AdversarialMatrix(t *testing.T) {
|
||||
good := []string{"/dev/sda", "/dev/sdb", "/dev/nvme0n1", "/dev/vda"}
|
||||
for _, d := range good {
|
||||
if err := ValidateSMARTDevice(d); err != nil {
|
||||
t.Errorf("ValidateSMARTDevice(%q) rejected a valid device: %v", d, err)
|
||||
}
|
||||
}
|
||||
bad := []string{
|
||||
"/dev/sda1", // a partition, not the whole disk (smartctl targets the disk)
|
||||
"/dev/../etc/shadow", // traversal
|
||||
"/dev/sda;reboot", // metacharacter
|
||||
"/dev/sda /dev/sdb", // extra arg
|
||||
"/etc/passwd", // not /dev
|
||||
"sda", // no /dev prefix
|
||||
"/dev/mapper/pve-root", // device-mapper not whitelisted
|
||||
"", // empty
|
||||
}
|
||||
for _, d := range bad {
|
||||
if err := ValidateSMARTDevice(d); err == nil {
|
||||
t.Errorf("ValidateSMARTDevice(%q) ACCEPTED a hostile device", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLVMName_AdversarialMatrix(t *testing.T) {
|
||||
for _, n := range []string{"pve", "data", "vg0", "vg.thin_pool"} {
|
||||
if err := ValidateLVMName(n); err != nil {
|
||||
t.Errorf("ValidateLVMName(%q) rejected a valid name: %v", n, err)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"", "-rf", "vg;reboot", "vg/pool extra", "vg\nx", "vg$(x)"} {
|
||||
if err := ValidateLVMName(n); err == nil {
|
||||
t.Errorf("ValidateLVMName(%q) ACCEPTED a hostile name", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostOps_RejectsHostileArgsBeforeExec is the proof that validation happens BEFORE any
|
||||
// command is constructed: a hostile UUID / mount path / device → error AND zero runner calls.
|
||||
func TestHostOps_RejectsHostileArgsBeforeExec(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("EnsureMount hostile UUID", func(t *testing.T) {
|
||||
rr := &recordingRunner{}
|
||||
ops := newTestHostOps(rr)
|
||||
err := ops.EnsureMount(ctx, MountSpec{Name: "x", UUID: "abcd; rm -rf /", Where: "/mnt/x"})
|
||||
if err == nil {
|
||||
t.Fatal("expected rejection")
|
||||
}
|
||||
if len(rr.calls) != 0 {
|
||||
t.Fatalf("a hostile UUID must be refused before any exec; got calls %v", rr.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("EnsureMount traversal mountpoint", func(t *testing.T) {
|
||||
rr := &recordingRunner{}
|
||||
ops := newTestHostOps(rr)
|
||||
err := ops.EnsureMount(ctx, MountSpec{Name: "x", UUID: "1234-ABCD", Where: "/mnt/../etc"})
|
||||
if err == nil || len(rr.calls) != 0 {
|
||||
t.Fatalf("traversal mountpoint must be refused before exec; err=%v calls=%v", err, rr.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("EnsureMount injection via mount options", func(t *testing.T) {
|
||||
rr := &recordingRunner{}
|
||||
ops := newTestHostOps(rr)
|
||||
err := ops.EnsureMount(ctx, MountSpec{Name: "x", UUID: "1234-ABCD", Where: "/mnt/x", Options: "ro\nWhat=/dev/sda"})
|
||||
if err == nil || len(rr.calls) != 0 {
|
||||
t.Fatalf("newline-injecting options must be refused before exec; err=%v calls=%v", err, rr.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SMART hostile device", func(t *testing.T) {
|
||||
rr := &recordingRunner{}
|
||||
ops := newTestHostOps(rr)
|
||||
_, err := ops.SMART(ctx, "/dev/sda;reboot")
|
||||
if err == nil || len(rr.calls) != 0 {
|
||||
t.Fatalf("hostile smart device must be refused before exec; err=%v calls=%v", err, rr.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ThinPoolMetadata hostile vg", func(t *testing.T) {
|
||||
rr := &recordingRunner{}
|
||||
ops := newTestHostOps(rr)
|
||||
if _, ok := ops.ThinPoolMetadata(ctx, "vg;reboot", "data"); ok {
|
||||
t.Fatal("hostile vg must return ok=false")
|
||||
}
|
||||
if len(rr.calls) != 0 {
|
||||
t.Fatalf("hostile vg must be refused before exec; calls=%v", rr.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// newTestHostOps builds a SudoHostOps over a recording runner with a temp stage dir (so the
|
||||
// EnsureMount staging write — which happens AFTER validation — has somewhere to go in the
|
||||
// rare valid-path test; hostile-path tests never reach it).
|
||||
func newTestHostOps(rr proxmox.Runner) *SudoHostOps {
|
||||
return NewSudoHostOps(SudoHostOpsConfig{
|
||||
Runner: rr,
|
||||
UnitDir: "/tmp/felhom-test-units",
|
||||
StageDir: testStageDir(),
|
||||
Logger: quietLogger(),
|
||||
})
|
||||
}
|
||||
|
||||
func TestSystemdEscapePath(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"/mnt/usb-backup": "mnt-usb\\x2dbackup",
|
||||
"/var/lib/vz": "var-lib-vz",
|
||||
"/srv/data": "srv-data",
|
||||
"/": "-",
|
||||
"/etc/foo.conf": "etc-foo.conf",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := systemdEscapePath(in); got != want {
|
||||
t.Errorf("systemdEscapePath(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// The unit name is derived deterministically and ends in .mount.
|
||||
name, err := UnitNameForMount("/mnt/usb-backup")
|
||||
if err != nil || !strings.HasSuffix(name, ".mount") {
|
||||
t.Errorf("UnitNameForMount = %q, %v", name, err)
|
||||
}
|
||||
}
|
||||
+132
-69
@@ -23,6 +23,7 @@ type KnownTarget struct {
|
||||
Name string
|
||||
Type string
|
||||
DurableID string
|
||||
UUID string // fs-UUID (mount-backed targets) — the by-UUID re-mount key
|
||||
Network bool // nfs/cifs/pbs — liveness is a reachability dial, not a device check
|
||||
MountBacked bool // usb/local-dir — a drop = its mountpoint disappears
|
||||
BackingDevice string // resolved block device (local targets)
|
||||
@@ -36,11 +37,23 @@ type KnownTargets interface {
|
||||
Known(ctx context.Context) ([]KnownTarget, error)
|
||||
}
|
||||
|
||||
// TargetLiveness reports whether one known target is presently up. Production is
|
||||
// HostLiveness (device/mount presence + a reachability dial, all non-privileged); tests
|
||||
// inject a fake.
|
||||
// TargetLiveness reports a known target's liveness. Production is HostLiveness (device/mount
|
||||
// presence + a reachability dial, all non-privileged); tests inject a fake.
|
||||
type TargetLiveness interface {
|
||||
// Present is the "in service" signal: mounted + reachable.
|
||||
Present(ctx context.Context, t KnownTarget) bool
|
||||
// DevicePresent is the "backing device is physically back" signal, independent of
|
||||
// whether it is mounted — the trigger for a benign re-mount of a returned drive.
|
||||
DevicePresent(ctx context.Context, t KnownTarget) bool
|
||||
}
|
||||
|
||||
// Remounter performs the benign re-mount response when a known mount-backed target's device
|
||||
// returns but its mountpoint is missing. The watchdog dispatches to it OFF its poll path
|
||||
// (a goroutine), never synchronously under the lock. Production routes through the gate
|
||||
// (benign) then HostOps.EnsureMount; wired in main.go so storage stays decoupled from
|
||||
// reconcile. A nil Remounter disables the response (observe-only, Phase-A behaviour).
|
||||
type Remounter interface {
|
||||
Remount(ctx context.Context, t KnownTarget)
|
||||
}
|
||||
|
||||
// Transition is one observed state change for a known target (for logging/diagnostics).
|
||||
@@ -57,30 +70,34 @@ type Transition struct {
|
||||
// It NEVER mutates anything (Phase A is read-only) — the benign re-mount-by-UUID response
|
||||
// to a return lands in Phase B. Here it only observes and signals.
|
||||
type Watchdog struct {
|
||||
targets KnownTargets
|
||||
liveness TargetLiveness
|
||||
interval time.Duration
|
||||
debounce time.Duration
|
||||
trigger func() // request an out-of-band report (debounced by the watchdog)
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
targets KnownTargets
|
||||
liveness TargetLiveness
|
||||
remounter Remounter // may be nil (observe-only)
|
||||
interval time.Duration
|
||||
debounce time.Duration
|
||||
trigger func() // request an out-of-band report (debounced by the watchdog)
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
spawn func(func()) // spawn a background task (overridable in tests; default `go f()`)
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
lastRemount map[string]time.Time // name -> last re-mount dispatch (rate-limit)
|
||||
}
|
||||
|
||||
// WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
|
||||
// rest default.
|
||||
// rest default. Remounter is optional (nil = observe-only).
|
||||
type WatchdogOptions struct {
|
||||
Targets KnownTargets
|
||||
Liveness TargetLiveness
|
||||
Trigger func()
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
Targets KnownTargets
|
||||
Liveness TargetLiveness
|
||||
Remounter Remounter
|
||||
Trigger func()
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
|
||||
@@ -103,14 +120,17 @@ func NewWatchdog(opts WatchdogOptions) *Watchdog {
|
||||
trigger = func() {}
|
||||
}
|
||||
return &Watchdog{
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
last: map[string]bool{},
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
remounter: opts.Remounter,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
spawn: func(f func()) { go f() },
|
||||
last: map[string]bool{},
|
||||
lastRemount: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,9 +157,11 @@ func (w *Watchdog) Run(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// tick performs one poll: read the known set, probe each target's liveness, diff against
|
||||
// the last-seen state, and fire a debounced trigger on any transition for a SEEN target.
|
||||
// It is deterministic given w.now — tests drive it directly with a fake clock.
|
||||
// tick performs one poll. Structure: probe liveness OUTSIDE the lock (the probes do IO —
|
||||
// mount reads, dials), then take the lock only for the state diff + debounce decision, then
|
||||
// perform side-effects (report trigger, re-mount dispatch) AFTER unlocking. The re-mount is
|
||||
// handed to a background task — never run synchronously under the lock or on the poll path.
|
||||
// Deterministic given w.now — tests drive it directly with a fake clock.
|
||||
func (w *Watchdog) tick(ctx context.Context) {
|
||||
known, err := w.targets.Known(ctx)
|
||||
if err != nil {
|
||||
@@ -147,52 +169,78 @@ func (w *Watchdog) tick(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
var transitions []Transition
|
||||
current := make(map[string]bool, len(known))
|
||||
for _, k := range known {
|
||||
present := w.liveness.Present(ctx, k)
|
||||
current[k.Name] = present
|
||||
prev, seen := w.last[k.Name]
|
||||
if !seen {
|
||||
continue // first observation → baseline only (never flag a never-attached drop)
|
||||
}
|
||||
if prev != present {
|
||||
transitions = append(transitions, Transition{Name: k.Name, From: stateStr(prev), To: stateStr(present)})
|
||||
}
|
||||
// Probe outside the lock.
|
||||
type probe struct {
|
||||
t KnownTarget
|
||||
present bool
|
||||
devicePresent bool
|
||||
}
|
||||
probes := make([]probe, 0, len(known))
|
||||
for _, k := range known {
|
||||
p := probe{t: k, present: w.liveness.Present(ctx, k)}
|
||||
if k.MountBacked && !p.present {
|
||||
p.devicePresent = w.liveness.DevicePresent(ctx, k)
|
||||
}
|
||||
probes = append(probes, p)
|
||||
}
|
||||
// Replace the baseline with the current snapshot (targets no longer known drop out).
|
||||
w.last = current
|
||||
|
||||
now := w.now()
|
||||
if len(transitions) > 0 {
|
||||
for _, tr := range transitions {
|
||||
w.logger.Warn("storage: watchdog detected target state change",
|
||||
"target", tr.Name, "from", tr.From, "to", tr.To)
|
||||
|
||||
w.mu.Lock()
|
||||
var transitions []Transition
|
||||
var remounts []KnownTarget
|
||||
current := make(map[string]bool, len(probes))
|
||||
for _, p := range probes {
|
||||
current[p.t.Name] = p.present
|
||||
if prev, seen := w.last[p.t.Name]; seen && prev != p.present {
|
||||
transitions = append(transitions, Transition{Name: p.t.Name, From: stateStr(prev), To: stateStr(p.present)})
|
||||
}
|
||||
// Re-mount candidate: a mount-backed target that is NOT mounted but whose backing
|
||||
// device is physically present (a disconnected→device-back state). Rate-limited per
|
||||
// target to the debounce window so a persistent mount failure can't storm HostOps.
|
||||
if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent {
|
||||
if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.debounce {
|
||||
w.lastRemount[p.t.Name] = now
|
||||
remounts = append(remounts, p.t)
|
||||
}
|
||||
}
|
||||
// Once a target is present again, clear its re-mount rate-limit so a future cycle
|
||||
// re-mounts promptly.
|
||||
if p.present {
|
||||
delete(w.lastRemount, p.t.Name)
|
||||
}
|
||||
}
|
||||
w.last = current // targets no longer known drop out
|
||||
|
||||
doFire := false
|
||||
if len(transitions) > 0 {
|
||||
if !w.fired || now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, len(transitions))
|
||||
doFire = true
|
||||
w.lastFire, w.fired, w.pending = now, true, false
|
||||
} else {
|
||||
w.pending = true
|
||||
w.logger.Debug("storage: watchdog debouncing transition", "pending_until", w.lastFire.Add(w.debounce))
|
||||
}
|
||||
return
|
||||
} else if w.pending && now.Sub(w.lastFire) >= w.debounce {
|
||||
doFire = true
|
||||
w.lastFire, w.pending = now, false
|
||||
}
|
||||
// No new transition, but a debounced one is pending and the window has elapsed → fire.
|
||||
if w.pending && now.Sub(w.lastFire) >= w.debounce {
|
||||
w.fire(now, 0)
|
||||
}
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
// fire requests the out-of-band report and resets the debounce window. Called under w.mu.
|
||||
func (w *Watchdog) fire(now time.Time, n int) {
|
||||
w.lastFire = now
|
||||
w.fired = true
|
||||
w.pending = false
|
||||
w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", n)
|
||||
w.trigger()
|
||||
// Side-effects, off the lock.
|
||||
for _, tr := range transitions {
|
||||
w.logger.Warn("storage: watchdog detected target state change",
|
||||
"target", tr.Name, "from", tr.From, "to", tr.To)
|
||||
}
|
||||
if doFire {
|
||||
w.logger.Info("storage: watchdog triggering out-of-band host-report", "transitions", len(transitions))
|
||||
w.trigger()
|
||||
}
|
||||
for _, t := range remounts {
|
||||
t := t
|
||||
w.logger.Info("storage: watchdog dispatching benign re-mount (device returned)",
|
||||
"target", t.Name, "where", t.MountPath)
|
||||
w.spawn(func() { w.remounter.Remount(ctx, t) })
|
||||
}
|
||||
}
|
||||
|
||||
func stateStr(present bool) string {
|
||||
@@ -250,6 +298,21 @@ func (h *HostLiveness) Present(ctx context.Context, t KnownTarget) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// DevicePresent reports whether the backing device is physically present (regardless of
|
||||
// mount state) — the re-mount trigger. Checks /dev/disk/by-uuid/<UUID> first (the by-UUID
|
||||
// link appears when the drive is plugged), then any known backing-device node.
|
||||
func (h *HostLiveness) DevicePresent(ctx context.Context, t KnownTarget) bool {
|
||||
if !t.MountBacked {
|
||||
return false
|
||||
}
|
||||
if t.UUID != "" {
|
||||
if dev, err := ByUUIDDevicePath(t.UUID); err == nil && h.host.DeviceExists(dev) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return t.BackingDevice != "" && h.host.DeviceExists(t.BackingDevice)
|
||||
}
|
||||
|
||||
func (h *HostLiveness) mounted(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
|
||||
@@ -24,10 +24,11 @@ func (s *staticKnown) Known(context.Context) ([]KnownTarget, error) {
|
||||
return s.targets, s.err
|
||||
}
|
||||
|
||||
// mapLiveness is a settable per-target presence fake.
|
||||
// mapLiveness is a settable per-target presence + device-presence fake.
|
||||
type mapLiveness struct {
|
||||
mu sync.Mutex
|
||||
present map[string]bool
|
||||
device map[string]bool // backing-device presence (re-mount trigger)
|
||||
}
|
||||
|
||||
func (m *mapLiveness) set(name string, p bool) {
|
||||
@@ -35,11 +36,24 @@ func (m *mapLiveness) set(name string, p bool) {
|
||||
defer m.mu.Unlock()
|
||||
m.present[name] = p
|
||||
}
|
||||
func (m *mapLiveness) setDevice(name string, p bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.device == nil {
|
||||
m.device = map[string]bool{}
|
||||
}
|
||||
m.device[name] = p
|
||||
}
|
||||
func (m *mapLiveness) Present(_ context.Context, t KnownTarget) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.present[t.Name]
|
||||
}
|
||||
func (m *mapLiveness) DevicePresent(_ context.Context, t KnownTarget) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.device[t.Name]
|
||||
}
|
||||
|
||||
// newTestWatchdog builds a watchdog with a manual clock and a trigger counter.
|
||||
func newTestWatchdog(known KnownTargets, live TargetLiveness, debounce time.Duration) (*Watchdog, *int, *time.Time) {
|
||||
@@ -129,6 +143,71 @@ func TestWatchdog_DebounceCoalescesFlaps(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRemounter records re-mount dispatches.
|
||||
type fakeRemounter struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (r *fakeRemounter) Remount(_ context.Context, t KnownTarget) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls = append(r.calls, t.Name)
|
||||
}
|
||||
func (r *fakeRemounter) count() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return len(r.calls)
|
||||
}
|
||||
|
||||
func TestWatchdog_ReMountOnDeviceReturn(t *testing.T) {
|
||||
known := &staticKnown{targets: []KnownTarget{{Name: "usb", MountBacked: true, UUID: "1234-ABCD", MountPath: "/mnt/usb"}}}
|
||||
live := &mapLiveness{present: map[string]bool{"usb": true}, device: map[string]bool{"usb": true}}
|
||||
rem := &fakeRemounter{}
|
||||
w, _, clock := newTestWatchdog(known, live, 30*time.Second)
|
||||
w.remounter = rem
|
||||
w.spawn = func(f func()) { f() } // run the dispatch synchronously for deterministic assertion
|
||||
ctx := context.Background()
|
||||
|
||||
w.tick(ctx) // baseline: present
|
||||
if rem.count() != 0 {
|
||||
t.Fatalf("no re-mount at baseline, got %d", rem.count())
|
||||
}
|
||||
|
||||
// Drop: device gone, unmounted. No re-mount (nothing to mount).
|
||||
live.set("usb", false)
|
||||
live.setDevice("usb", false)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 0 {
|
||||
t.Fatalf("no re-mount while device absent, got %d", rem.count())
|
||||
}
|
||||
|
||||
// Device returns but still unmounted → re-mount dispatched.
|
||||
live.setDevice("usb", true)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 1 {
|
||||
t.Fatalf("re-mount expected when device returns unmounted, got %d", rem.count())
|
||||
}
|
||||
|
||||
// Still device-present-unmounted within the debounce window → rate-limited (no storm).
|
||||
*clock = clock.Add(5 * time.Second)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 1 {
|
||||
t.Fatalf("re-mount must be rate-limited within debounce, got %d", rem.count())
|
||||
}
|
||||
|
||||
// Successful mount (present=true) clears the rate-limit; a later cycle re-mounts again.
|
||||
live.set("usb", true)
|
||||
*clock = clock.Add(5 * time.Second)
|
||||
w.tick(ctx) // present → clears lastRemount
|
||||
live.set("usb", false) // drop again, device still present
|
||||
*clock = clock.Add(5 * time.Second)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 2 {
|
||||
t.Fatalf("a fresh device cycle should re-mount again, got %d", rem.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_ReadErrorSkipsTick(t *testing.T) {
|
||||
known := &staticKnown{err: errors.New("proxmox blip")}
|
||||
live := &mapLiveness{present: map[string]bool{}}
|
||||
|
||||
Reference in New Issue
Block a user