v0.5.0: slice 5 Phase B — the host-root surface (mounts + SMART + grow + destructive gate)

The privileged write surface, isolated behind a narrow, arg-validated, adversarially- tested seam (HostOps), the same discipline as the slice-4 gate. Completes slice 5. - internal/storage: HostOps seam + SudoHostOps (systemd .mount units by fs-UUID, detach, SMART, lvs) via sudoers allowlist + fixed arg vectors, no shell; NoopHostOps fallback. - validate.go: strict UUID/mount-path/device/LVM validators + in-process systemd-escape. Headline test: adversarial matrix (metacharacters/traversal/malformed) refused with zero exec. - smart.go: smartctl SATA + NVMe parse, UNKNOWN-degrade; lvs thin-pool metadata fill. - observer enrichment (Observe only): fills smart + thin-pool metadata. - watchdog: benign re-mount response off the poll path (DevicePresent probe, rate-limited). - reconcile: ActionResize (benign, grow-only) + proxmox.ResizeLXC; destructive storage ops (ClassStorageWipe/Decommission) through the slice-4 gate, target-scoped; built+tested, inert live. - --selftest=storage [-watch] live harness; configs/felhom-agent.sudoers; privileged.* knobs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 10:53:38 +02:00
parent 27b68f043b
commit 9d6e49236c
25 changed files with 2074 additions and 182 deletions
@@ -16,6 +16,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"

@@ -30,18 +31,20 @@ import (

 // version is the agent version. Overridable at build time with
 // -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
-var version = "0.5.0-rc1"
+var version = "0.5.0"

 func main() {
 	var (
 		cfgPath     string
 		selftest    selftestFlag
 		vmid        int
+		watch       time.Duration
 		showVersion bool
 	)
 	flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)")
-	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report to the hub")
+	flag.Var(&selftest, "selftest", "run a self-test and exit: bare/`read` = read-only queries; `task` = reversible mutating exercise (needs -vmid); `hub` = one collect+report to the hub; `storage` = observe storage targets (+ -watch for the live watchdog)")
 	flag.IntVar(&vmid, "vmid", 0, "guest VMID for --selftest=task (the reversible snapshot/rollback exercise)")
+	flag.DurationVar(&watch, "watch", 0, "for --selftest=storage: run the watchdog verbose for this duration (e.g. 3m) with the re-mount response live; 0 = observe pass only")
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.Parse()

@@ -71,6 +74,8 @@ func main() {
 		os.Exit(runSelftestTask(context.Background(), cfg, logger, vmid))
 	case "hub":
 		os.Exit(runSelftestHub(context.Background(), cfg, logger))
+	case "storage":
+		os.Exit(runSelftestStorage(context.Background(), cfg, logger, watch))
 	}
 }

@@ -88,6 +93,62 @@ func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) {
 	})
 }

+// newHostOps builds the privileged storage surface (slice 5 Phase B) from config. It shells
+// out via the same fenced Runner the proxmox layer uses (sudo -n, arg vectors, no shell);
+// every argument is validated in internal/storage before any command is built. A
+// missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a
+// crash.
+func newHostOps(cfg config.Config, logger *slog.Logger) storage.HostOps {
+	mode := proxmox.RunnerMode(cfg.Privileged.Mode)
+	if mode == "" {
+		mode = proxmox.RunnerSudo
+	}
+	runner := &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath}
+	return storage.NewSudoHostOps(storage.SudoHostOpsConfig{
+		Runner: runner,
+		Bins: storage.Binaries{
+			Systemctl: cfg.Privileged.Systemctl,
+			Install:   cfg.Privileged.Install,
+			Smartctl:  cfg.Privileged.Smartctl,
+			Lvs:       cfg.Privileged.Lvs,
+		},
+		UnitDir:  cfg.Privileged.UnitDir,
+		StageDir: cfg.Privileged.StageDir,
+		Logger:   logger,
+	})
+}
+
+// gateRemounter is the watchdog's benign re-mount response: it routes the re-mount through
+// the reversibility gate (classified benign) and, if allowed, calls HostOps.EnsureMount. It
+// lives here (not in internal/storage) so storage stays decoupled from reconcile — main is
+// the one place that holds both.
+type gateRemounter struct {
+	gate   *reconcile.Gate
+	ops    storage.HostOps
+	hostID string
+	logger *slog.Logger
+}
+
+// Remount authorizes (benign) and performs a by-UUID re-mount of a returned target.
+func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
+	dec := r.gate.Authorize(reconcile.IntentForStorageMount(r.hostID, t.Name), nil)
+	if !dec.Allowed {
+		r.logger.Warn("storage: re-mount refused by gate (unexpected for a benign mount)",
+			"target", t.Name, "reason", dec.Reason)
+		return
+	}
+	uuid := t.UUID
+	if uuid == "" {
+		uuid = strings.TrimPrefix(t.DurableID, "uuid:") // durable_id carries it for usb/local-dir
+	}
+	spec := storage.MountSpec{Name: t.Name, UUID: uuid, Where: t.MountPath}
+	if err := r.ops.EnsureMount(ctx, spec); err != nil {
+		r.logger.Error("storage: re-mount failed", "target", t.Name, "where", t.MountPath, "err", err)
+		return
+	}
+	r.logger.Info("storage: re-mounted returned target", "target", t.Name, "where", t.MountPath)
+}
+
 // runDaemon is the default mode: collect a host-report and POST it to the hub on a
 // loop. Requires both proxmox (to collect) and hub config.
 func runDaemon(cfg config.Config, logger *slog.Logger) int {
@@ -110,34 +171,17 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 		return 1
 	}
 	hcfg := cfg.Hub.WithDefaults()
-	// Storage observer (slice 5): read-only, builds the report's storage_targets from
-	// Proxmox + non-privileged host reads. Wired into the collector via the StorageObserver
-	// seam (so hub does not import storage).
+	// Storage observer (slice 5): builds the report's storage_targets from Proxmox +
+	// non-privileged host reads, enriched with the privileged SMART/lvs reads via HostOps
+	// (Phase B). Wired into the collector via the StorageObserver seam (so hub does not
+	// import storage).
 	hostReader := storage.NewProcHostReader()
-	observer := storage.NewObserver(px, hostReader, logger)
+	hostOps := newHostOps(cfg, logger)
+	observer := storage.NewObserver(px, hostReader, hostOps, logger)
 	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, cfg.Hub.HostID, version, logger)
 	loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
 	interval := time.Duration(hcfg.PollSeconds) * time.Second

-	// Storage watchdog (slice 5): a third daemon goroutine fast-polling the known target
-	// set for attached↔disconnected transitions and triggering an immediate, debounced
-	// out-of-band report. With no removable/network storage it simply finds nothing to flag.
-	storageTrigger := make(chan struct{}, 1)
-	loop.SetTrigger(storageTrigger)
-	watchdog := storage.NewWatchdog(storage.WatchdogOptions{
-		Targets:  storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
-		Liveness: storage.NewHostLiveness(hostReader, 0),
-		Trigger: func() {
-			select {
-			case storageTrigger <- struct{}{}:
-			default:
-			}
-		},
-		Interval: cfg.Storage.WatchdogInterval(),
-		Debounce: cfg.Storage.WatchdogDebounce(),
-		Logger:   logger,
-	})
-
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 	logger.Info("felhom-agent daemon starting",
@@ -178,6 +222,29 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
 	}
 	gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)

+	// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
+	// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
+	// known mount-backed target's device returning unmounted, dispatches a benign re-mount
+	// (routed through the gate as benign, then HostOps). With no removable/network storage
+	// it finds nothing to flag. The re-mount dispatch is off the poll path (a goroutine).
+	storageTrigger := make(chan struct{}, 1)
+	loop.SetTrigger(storageTrigger)
+	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
+	watchdog := storage.NewWatchdog(storage.WatchdogOptions{
+		Targets:   storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
+		Liveness:  storage.NewHostLiveness(hostReader, 0),
+		Remounter: remounter,
+		Trigger: func() {
+			select {
+			case storageTrigger <- struct{}{}:
+			default:
+			}
+		},
+		Interval: cfg.Storage.WatchdogInterval(),
+		Debounce: cfg.Storage.WatchdogDebounce(),
+		Logger:   logger,
+	})
+
 	engine := reconcile.NewEngine(reconcile.EngineOptions{
 		API:      px,
 		Queue:    queue,
@@ -270,7 +337,7 @@ func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger)
 		fmt.Fprintln(os.Stderr, "selftest: hub client:", err)
 		return 1
 	}
-	observer := storage.NewObserver(px, storage.NewProcHostReader(), logger)
+	observer := storage.NewObserver(px, storage.NewProcHostReader(), newHostOps(cfg, logger), logger)
 	collector := hub.NewCollector(px, hub.SystemctlProber{}, observer, cfg.Hub.HostID, version, logger)

 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
@@ -299,6 +366,100 @@ func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger)
 	return 0
 }

+// runSelftestStorage is the live storage harness (slice 5 Phase B, for the USB runbook).
+// It needs Proxmox config only (NO hub) so it runs standalone on the Proxmox host:
+//   - observe pass: print the full StorageTarget table incl. the privileged SMART summary
+//     and thin-pool data+metadata fill.
+//   - -watch D: run the watchdog verbose for D with the re-mount response LIVE, so the
+//     operator can physically cycle a drive and watch detect → report → re-mount.
+func runSelftestStorage(ctx context.Context, cfg config.Config, logger *slog.Logger, watch time.Duration) int {
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox not configured:", err)
+		return 1
+	}
+	px, err := newProxmoxClient(cfg)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
+		return 1
+	}
+	hostReader := storage.NewProcHostReader()
+	hostOps := newHostOps(cfg, logger)
+	observer := storage.NewObserver(px, hostReader, hostOps, logger)
+
+	octx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	fmt.Printf("=== felhom-agent %s selftest=storage ===\n", version)
+	targets, err := observer.Observe(octx)
+	cancel()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "  [FAIL] observe:", err)
+		return 1
+	}
+	fmt.Printf("  observed %d storage target(s):\n", len(targets))
+	for _, t := range targets {
+		fmt.Printf("   - %-12s type=%-9s state=%-12s reach=%-5v class=%-4s durable=%s\n",
+			t.Name, t.Type, t.State, t.Reachable, t.ClassHint, t.DurableID)
+		fmt.Printf("       usage %s/%s (%.0f%%)  mount=%q dev=%q\n",
+			gib(t.UsedBytes), gib(t.TotalBytes), t.UsedFraction*100, t.MountPath, t.BackingDevice)
+		fmt.Printf("       smart: health=%s%s\n", t.Smart.Health, smartCounters(t.Smart))
+		if t.ThinPool != nil {
+			meta := "n/a"
+			if t.ThinPool.MetadataUsedFraction != nil {
+				meta = fmt.Sprintf("%.1f%%", *t.ThinPool.MetadataUsedFraction*100)
+			}
+			fmt.Printf("       thin-pool: data=%.1f%% metadata=%s\n", t.ThinPool.DataUsedFraction*100, meta)
+		}
+	}
+
+	if watch <= 0 {
+		fmt.Println("=== selftest=storage OK (observe pass; pass -watch D for the live watchdog) ===")
+		return 0
+	}
+
+	// Live watchdog window: re-mount response active. Cycle a drive and watch the logs.
+	fmt.Printf("  --- watching for %s (cycle a drive now; detect → report → re-mount) ---\n", watch)
+	wctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	wctx, cancel2 := context.WithTimeout(wctx, watch)
+	defer cancel2()
+
+	gate := reconcile.NewGate(nil, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
+	remounter := &gateRemounter{gate: gate, ops: hostOps, hostID: cfg.Hub.HostID, logger: logger}
+	wd := storage.NewWatchdog(storage.WatchdogOptions{
+		Targets:   storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
+		Liveness:  storage.NewHostLiveness(hostReader, 0),
+		Remounter: remounter,
+		Trigger:   func() { logger.Info("storage: (selftest) would send out-of-band host-report now") },
+		Interval:  cfg.Storage.WatchdogInterval(),
+		Debounce:  cfg.Storage.WatchdogDebounce(),
+		Logger:    logger,
+	})
+	_ = wd.Run(wctx)
+	fmt.Println("=== selftest=storage watch window ended ===")
+	return 0
+}
+
+// smartCounters renders the non-nil SMART counters compactly for the selftest table.
+func smartCounters(s hub.SmartSummary) string {
+	var parts []string
+	add := func(name string, v *int) {
+		if v != nil {
+			parts = append(parts, fmt.Sprintf("%s=%d", name, *v))
+		}
+	}
+	add("temp", s.TemperatureC)
+	add("poh", s.PowerOnHours)
+	add("realloc", s.ReallocatedSectors)
+	add("pending", s.PendingSectors)
+	add("offline_unc", s.OfflineUncorrectable)
+	add("crit_warn", s.CriticalWarning)
+	add("media_err", s.MediaErrors)
+	add("pct_used", s.PercentageUsed)
+	if len(parts) == 0 {
+		return ""
+	}
+	return " " + strings.Join(parts, " ")
+}
+
 // runSelftestRead loads config, builds the API client, and runs the read-only
 // queries against the live host, printing a short health report. It mutates
 // nothing. Missing/invalid config is reported cleanly (no panic).
@@ -585,8 +746,10 @@ func (f *selftestFlag) Set(v string) error {
 		f.mode = "task"
 	case "hub":
 		f.mode = "hub"
+	case "storage":
+		f.mode = "storage"
 	default:
-		return fmt.Errorf("invalid --selftest value %q (want read|task|hub)", v)
+		return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage)", v)
 	}
 	return nil
 }