From 63aa63d0d69c2551881acc1e42160fc7fd934d18 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 30 Jun 2026 11:12:27 +0200 Subject: [PATCH] agent v0.50.0: NAS network storage Part A1 (NFS/SMB automount foundation) Host-side NFS/SMB automount of a bulk-media NAS share under /mnt/felhom-drives/ (propagates into the guest via the existing shared bind), the +100000 uid recipe, per-share liveness, and add/list/remove local-API endpoints. A NAS is a distinct class that bypasses the drive enroll/eject/decommission/SMART/watchdog machinery. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs --- CHANGELOG.md | 43 +++ cmd/felhom-agent/main.go | 14 +- configs/felhom-agent.sudoers | 19 +- internal/config/config.go | 4 + internal/localapi/netstorage.go | 204 +++++++++++ internal/localapi/netstorage_test.go | 202 +++++++++++ internal/localapi/server.go | 18 + internal/storage/mountunit.go | 7 + internal/storage/netmount.go | 511 +++++++++++++++++++++++++++ internal/storage/netmount_test.go | 334 +++++++++++++++++ 10 files changed, 1351 insertions(+), 5 deletions(-) create mode 100644 internal/localapi/netstorage.go create mode 100644 internal/localapi/netstorage_test.go create mode 100644 internal/storage/netmount.go create mode 100644 internal/storage/netmount_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8244463..ef130fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,46 @@ +## v0.50.0 — NAS network storage Part A1: NFS/SMB automount foundation (2026-06-30) + +Agent foundation of the validated `SPIKE-nas-storage-2026-06-29.md` (verdict READY): a customer NAS can +serve **bulk media** to a media app. The agent mounts a NAS share **host-side** under +`/mnt/felhom-drives/` via a systemd `.automount` (+ `.mount`) pair; it propagates into guest 9201 for +free through the existing shared `mp8` bind (no new mountpoint, no restart). A NAS is a **distinct storage +class** — it carries **no durable-id** and never enters the drive enroll/eject/decommission/wipe/SMART/ +watchdog machinery. **Bulk-media class only; STOP before A2 (controller registry/UI) + B (restic-SFTP).** + +- **`internal/storage/netmount.go` (NEW).** `NetworkMountSpec` + the locked SPIKE recipe: + - **NFS (preferred):** `What=server:/export`, `Type=nfs4`, `Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev`. + `soft` is the failure-isolation knob (clean EIO, never a `df`/guest wedge); a default `hard` mount is + never emitted. The `+100000` uid mapping is the **export's** job (`anonuid=101000`), so the client mount + carries no uid. + - **SMB (fallback):** `What=//server/share`, `Type=cifs`, + `Options=vers=3.0,credentials=<0600 file>,uid=<+100000>,gid=<+100000>,forceuid,forcegid,file_mode=0664,dir_mode=0775,_netdev` + (plain octal modes, never setgid 2775). **The +100000 rule** (container uid/gid N = host N+100000): a + container uid 1000 renders `uid=101000` so the guest sees its native id and reads+writes; a naïve `+0` + lands as `nobody:nogroup` (not writable) — the documented trap, asserted by a companion test. + - **`.automount` with `TimeoutIdleSec`** (on-demand + idle-unmount): an idle NAS reboot is a non-event. + - **per-share liveness** (`ListNetworkMounts`): TCP-probes the NAS endpoint (2049/445) + reads + `/proc/mounts` — it never `stat`s the (possibly EIO/D-state) mountpoint, so a black-holed NAS cannot + wedge a list. Health `ok | idle | unreachable`, scoped to the affected share, never box-wide. + - **role gate** `NetworkMountRole`: network storage is **bulk-userdata only** — confined to the + `/mnt/felhom-drives` namespace; any other target is refused (most-protected). + - Full validation (`ValidateNetworkMountSpec`) before any unit is rendered: share name (safe segment), + server, NFS export (absolute, no traversal) / SMB share name, uid/gid range, creds path. +- **Drive-machinery bypass (Scenario D).** `parseFelhomMountUnit` (the host-reboot drive re-assert's + classifier) explicitly refuses any unit carrying the network marker, so a NAS mount is never given a + durable-id, SMART-probed, or re-asserted as a drive. Companion red-proof: the same by-uuid-shaped unit + with the drive marker DOES parse — the guard is the discriminator, not luck. +- **`internal/localapi/netstorage.go` (NEW).** Self-scoped endpoints `POST /netstorage/add`, + `GET /netstorage`, `POST /netstorage/remove`. SMB credentials are written **out-of-band** to a 0600 file + the agent owns (never in git, never in a plaintext registry, never logged). Role-gated to the user-data + namespace. +- **sudoers:** new narrow `FELHOM_NETMOUNT` alias (install/enable/disable/stop the `.automount` + remove the + felhom mount-unit files; the `.mount` half reuses `FELHOM_MOUNT`, the mountpoint mkdir reuses + `FELHOM_INTERMEDIARY`). `visudo -cf` clean. +- **config:** `privileged.smb_creds_dir` (default `/var/lib/felhom-agent/smb-creds`). +- **Runtime deps:** `mount.nfs` (nfs-common) + `mount.cifs` (cifs-utils) present on the host (confirmed live). +- Tests: exact NFS/SMB option-set string-asserts + the +100000 companion; validation matrix; role gate; + unit round-trip + health; the drive-machinery guard + companion; Ensure/Remove command sequences. + ## v0.49.0 — reboot-during-backup stale-lock recovery (F2-b) + shared-parent script redeploy fix (F2-a) (2026-06-30) Closes the two host-reboot findings from `TESTRUN-fullstack-2026-06-29.md`. diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 3cc8c1a..6a46cd2 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -45,7 +45,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.49.0" +var version = "0.50.0" // runGuestHook is the PVE pre-start hook body (`felhom-agent guest-hook `). On the // pre-start phase it creates placeholder dirs for any absent bind-mount source so the guest always boots @@ -216,7 +216,10 @@ func newProxmoxClient(cfg config.Config) (*proxmox.Client, error) { // every argument is validated in internal/storage before any command is built. A // missing/declined sudoers entry degrades per-op (SMART→UNKNOWN, mount→logged error), not a // crash. -func newHostOps(cfg config.Config, logger *slog.Logger) storage.HostOps { +// Returns the concrete *storage.SudoHostOps (not the HostOps interface) so callers that need the +// methods outside that lean interface — the host-reboot mount re-assert and the Part-A1 network-mount +// surface — can reach them without a type assertion. It still satisfies storage.HostOps everywhere. +func newHostOps(cfg config.Config, logger *slog.Logger) *storage.SudoHostOps { mode := proxmox.RunnerMode(cfg.Privileged.Mode) if mode == "" { mode = proxmox.RunnerSudo @@ -549,7 +552,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { // re-enumeration can move the device (/dev/sdb→sdc); ReassertEnrolledMounts re-resolves each by // filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op. // Type-asserted (the concrete op exposes it; the interface stays lean). - mountReasserter, _ := hostOps.(*storage.SudoHostOps) + mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts if mountReasserter != nil { mountReasserter.ReassertEnrolledMounts(ctx) } @@ -699,7 +702,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re // leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the // daemon — the host still reports/reconciles; only the controller channel is unavailable until // fixed. The opened token store is returned via outTokens so the caller can Close it. -func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps storage.HostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { +func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server { if !cfg.LocalAPI.Enabled() { return nil } @@ -751,6 +754,9 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St DiskGate: storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID}, Guests2: px, GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest + // Network storage (NAS) — Part A1: the privileged host network-mount surface (NFS/SMB automount). + NetStorage: hostOps, + SmbCredsDir: cfg.Privileged.SmbCredsDir, ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap // F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start // go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent). diff --git a/configs/felhom-agent.sudoers b/configs/felhom-agent.sudoers index 9a03eff..2ef7655 100644 --- a/configs/felhom-agent.sudoers +++ b/configs/felhom-agent.sudoers @@ -127,4 +127,21 @@ Cmnd_Alias FELHOM_CONTROLLERSWAP = \ Cmnd_Alias FELHOM_STALELOCK = \ /usr/sbin/pct unlock [0-9]* -felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK +# Network storage / NAS (Part A1, SPIKE-nas-storage-2026-06-29). The agent mounts a customer NAS share +# HOST-SIDE under /mnt/felhom-drives/ via a systemd .automount (+ .mount) pair so it propagates +# into the guest through the existing shared bind (an unprivileged LXC cannot mount NFS/CIFS itself). +# A NAS is NOT a drive — no durable-id, no SMART, no wipe; these grants only install/enable/remove the +# unit pair. The agent fine-validates every value (share name, server, export, uid/gid, creds path) before +# any unit is rendered (internal/storage/netmount.go ValidateNetworkMountSpec); the trailing globs are the +# COARSE allowlist. The `.mount` install/enable/disable/stop reuse FELHOM_MOUNT; this alias adds the +# `.automount` variants + the unit-file removal. The unit FILE name is the systemd-escaped mountpoint, +# which always begins `mnt-felhom` (the mountpoint is /mnt/felhom-drives/), so the rm glob is scoped +# to felhom mount units only. mkdir of the mountpoint reuses FELHOM_INTERMEDIARY's /mnt/felhom-drives/*. +Cmnd_Alias FELHOM_NETMOUNT = \ + /usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/units/* /etc/systemd/system/*.automount, \ + /usr/bin/systemctl enable --now -- *.automount, \ + /usr/bin/systemctl disable -- *.automount, \ + /usr/bin/systemctl stop -- *.automount, \ + /usr/bin/rm -f /etc/systemd/system/mnt-felhom* + +felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT diff --git a/internal/config/config.go b/internal/config/config.go index b894e44..e269032 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -386,6 +386,10 @@ type PrivilegedConfig struct { Install string `json:"install"` // default /usr/bin/install Smartctl string `json:"smartctl"` // default /usr/sbin/smartctl Lvs string `json:"lvs"` // default /usr/sbin/lvs + + // SmbCredsDir is where the agent writes 0600 SMB credentials files for network storage (Part A1). + // Out-of-band: never committed, never logged. Default /var/lib/felhom-agent/smb-creds (agent-owned). + SmbCredsDir string `json:"smb_creds_dir"` } // Default returns a Config pre-populated with sane defaults. diff --git a/internal/localapi/netstorage.go b/internal/localapi/netstorage.go new file mode 100644 index 0000000..cf218a6 --- /dev/null +++ b/internal/localapi/netstorage.go @@ -0,0 +1,204 @@ +package localapi + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "gitea.dooplex.hu/admin/felhom-agent/internal/storage" +) + +// Network storage (NAS) — Part A1, doc 03 §6 + SPIKE-nas-storage-2026-06-29.md. The agent mounts a NAS +// share HOST-SIDE under /mnt/felhom-drives/ (it propagates into the guest via the existing shared +// bind); Docker then binds it into a media container. A NAS is a DISTINCT class from a physical drive: +// it bypasses the disk enroll/eject/decommission/wipe machinery entirely (no durable-id, no SMART). The +// controller registry/UI (A2) drives these endpoints; A1 is the agent foundation. +// +// Self-scoping is unchanged (withGuest): one customer per host, the token's guest authorizes the host's +// network-mount surface. The mount is bulk-userdata role ONLY (role-gated below). + +// NetworkStorageOps is the privileged network-mount surface the endpoints need. Satisfied by +// *storage.SudoHostOps. Optional — the endpoints report "not configured" when absent. +type NetworkStorageOps interface { + EnsureNetworkMount(ctx context.Context, spec storage.NetworkMountSpec) error + RemoveNetworkMount(ctx context.Context, name string) error + ListNetworkMounts(ctx context.Context) ([]storage.NetworkMountStatus, error) +} + +// defaultSmbCredsDir is where the agent writes the 0600 SMB credentials files (out-of-band; never in +// git, never in a plaintext registry, never logged — same posture as the rotated Resend key). +const defaultSmbCredsDir = "/var/lib/felhom-agent/smb-creds" + +// netStorageAddRequest is POST /netstorage/add. For SMB, username+password are written by the agent to a +// 0600 creds file and NEVER logged or echoed back; for NFS they are ignored (server-side squash handles +// uid mapping). mapped_uid/mapped_gid are the CONTAINER ids (the agent applies the +100000 host offset). +type netStorageAddRequest struct { + VMID int `json:"vmid"` // optional; must match the token's guest if set + Name string `json:"name"` + Protocol string `json:"protocol"` // "nfs" | "smb" + Server string `json:"server"` + Export string `json:"export"` // NFS export path | SMB share name + MappedUID int `json:"mapped_uid"` // container uid (e.g. 1000) + MappedGID int `json:"mapped_gid"` // container gid + IdleTimeoutSec int `json:"idle_timeout_sec"` // automount idle-unmount window; 0 → default + Username string `json:"username,omitempty"` // SMB only (secret — written to creds file) + Password string `json:"password,omitempty"` // SMB only (secret — written to creds file) +} + +// handleNetStorageAdd mounts a NAS share host-side. It role-gates the mount to the user-data namespace, +// writes the SMB creds file out-of-band (0600), then calls EnsureNetworkMount (automount idle-unmount). +func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmid int) { + if s.netStorage == nil { + writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") + return + } + var req netStorageAddRequest + if !decodeBody(w, r, &req) { + return + } + if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { + return + } + + // Role gate (SPIKE Q7): network storage is bulk-userdata ONLY. The mount lands under the + // network-mount root; refuse anything that would resolve outside the user-data namespace. + where := s.netMountRoot + "/" + strings.TrimSpace(req.Name) + if storage.NetworkMountRole(where) != storage.RoleUserData { + s.logger.Warn("local-api: network mount refused — not a user-data path", + "name", req.Name, "where", where, "role", storage.NetworkMountRole(where)) + writeErr(w, http.StatusForbidden, "network storage is restricted to the user-data namespace") + return + } + + spec := storage.NetworkMountSpec{ + Name: strings.TrimSpace(req.Name), + Protocol: storage.NetworkProtocol(strings.ToLower(strings.TrimSpace(req.Protocol))), + Server: strings.TrimSpace(req.Server), + Export: strings.TrimSpace(req.Export), + MappedUID: req.MappedUID, + MappedGID: req.MappedGID, + IdleTimeoutSec: req.IdleTimeoutSec, + } + + // SMB: stage the credentials out-of-band (0600) BEFORE the mount. NFS needs none (server squash). + if spec.Protocol == storage.ProtocolSMB { + credsPath, err := s.writeSMBCreds(spec.Name, req.Username, req.Password) + if err != nil { + s.logger.Error("local-api: writing SMB credentials failed", "name", spec.Name, "err", err) + writeErr(w, http.StatusBadRequest, "could not stage SMB credentials") + return + } + spec.CredsRef = credsPath + } + + if err := s.netStorage.EnsureNetworkMount(r.Context(), spec); err != nil { + s.logger.Error("local-api: network mount add failed", "name", spec.Name, "proto", spec.Protocol, "err", err) + // Clean up a creds file we staged for a mount that then failed to apply. + if spec.Protocol == storage.ProtocolSMB { + s.removeSMBCreds(spec.Name) + } + writeErr(w, http.StatusBadGateway, "network mount failed: "+err.Error()) + return + } + s.logger.Info("local-api: network mount added", "name", spec.Name, "proto", spec.Protocol, + "server", spec.Server, "export", spec.Export, "where", spec.Where()) + writeOK(w, map[string]any{ + "name": spec.Name, + "protocol": string(spec.Protocol), + "where": spec.Where(), + "host_uid": spec.HostUID(), + "host_gid": spec.HostGID(), + "guest_path": spec.Where(), // the in-guest path the controller (A2) repoints a media app's data dir to + }) +} + +// handleNetStorageList lists configured network mounts + per-share liveness (read-only/benign). +func (s *Server) handleNetStorageList(w http.ResponseWriter, r *http.Request, vmid int) { + if s.netStorage == nil { + writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") + return + } + mounts, err := s.netStorage.ListNetworkMounts(r.Context()) + if err != nil { + s.logger.Error("local-api: network mount list failed", "err", err) + writeErr(w, http.StatusBadGateway, "could not list network mounts") + return + } + if mounts == nil { + mounts = []storage.NetworkMountStatus{} + } + writeOK(w, map[string]any{"vmid": vmid, "network_mounts": mounts}) +} + +type netStorageRemoveRequest struct { + VMID int `json:"vmid"` + Name string `json:"name"` +} + +// handleNetStorageRemove unmounts + removes a network mount and its SMB creds file. It NEVER routes +// through the drive eject/decommission path — a NAS has no device lifecycle to tear down. +func (s *Server) handleNetStorageRemove(w http.ResponseWriter, r *http.Request, vmid int) { + if s.netStorage == nil { + writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") + return + } + var req netStorageRemoveRequest + if !decodeBody(w, r, &req) { + return + } + if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { + return + } + name := strings.TrimSpace(req.Name) + if name == "" { + writeErr(w, http.StatusBadRequest, "name is required") + return + } + if err := s.netStorage.RemoveNetworkMount(r.Context(), name); err != nil { + s.logger.Error("local-api: network mount remove failed", "name", name, "err", err) + writeErr(w, http.StatusBadGateway, "network mount removal failed: "+err.Error()) + return + } + s.removeSMBCreds(name) // best-effort: a NFS mount has no creds file (no-op) + s.logger.Info("local-api: network mount removed", "name", name) + writeOK(w, map[string]any{"name": name, "removed": true}) +} + +// ---- SMB credentials (out-of-band) --------------------------------------------------------------- + +// writeSMBCreds writes a 0600 mount.cifs credentials file for the share and returns its path. The file +// is owned by the agent user; root (the systemd-triggered mount.cifs) still reads it. The secret is +// NEVER logged. A missing username/password is an error (SMB needs credentials). +func (s *Server) writeSMBCreds(name, username, password string) (string, error) { + if username == "" || password == "" { + return "", fmt.Errorf("smb credentials (username + password) are required") + } + if strings.ContainsAny(username, "\n\r") || strings.ContainsAny(password, "\n\r") { + return "", fmt.Errorf("smb credentials must not contain newlines") + } + dir := s.smbCredsDir + if dir == "" { + dir = defaultSmbCredsDir + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("creds dir: %w", err) + } + path := filepath.Join(dir, name+".cred") + body := "username=" + username + "\npassword=" + password + "\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + return "", fmt.Errorf("writing creds: %w", err) + } + return path, nil +} + +// removeSMBCreds deletes a share's creds file (best-effort; absent is fine). +func (s *Server) removeSMBCreds(name string) { + dir := s.smbCredsDir + if dir == "" { + dir = defaultSmbCredsDir + } + _ = os.Remove(filepath.Join(dir, name+".cred")) +} diff --git a/internal/localapi/netstorage_test.go b/internal/localapi/netstorage_test.go new file mode 100644 index 0000000..82d4cfe --- /dev/null +++ b/internal/localapi/netstorage_test.go @@ -0,0 +1,202 @@ +package localapi + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/storage" +) + +// fakeNetOps records the network-mount surface calls (never touches a real host). +type fakeNetOps struct { + ensured []storage.NetworkMountSpec + removed []string + list []storage.NetworkMountStatus + ensErr error +} + +func (f *fakeNetOps) EnsureNetworkMount(_ context.Context, s storage.NetworkMountSpec) error { + f.ensured = append(f.ensured, s) + return f.ensErr +} +func (f *fakeNetOps) RemoveNetworkMount(_ context.Context, n string) error { + f.removed = append(f.removed, n) + return nil +} +func (f *fakeNetOps) ListNetworkMounts(_ context.Context) ([]storage.NetworkMountStatus, error) { + return f.list, nil +} + +func newNetServer(t *testing.T, n NetworkStorageOps, credsDir string) *Server { + t.Helper() + srv, err := NewServer(Options{ + ListenAddr: "127.0.0.1:0", + Guests: &fakeGuests{}, + Backups: &fakeBackups{}, + Store: &fakeStore{}, + Storage: fakeStorage{}, + Tokens: staticTokens{"A": 8200, "B": 9300}, + NetStorage: n, + SmbCredsDir: credsDir, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("new server: %v", err) + } + return srv +} + +func TestNetStorage_AddNFS_HappyPath(t *testing.T) { + n := &fakeNetOps{} + h := newNetServer(t, n, t.TempDir()).Handler() + body := `{"name":"media","protocol":"nfs","server":"10.0.0.5","export":"/srv/media","mapped_uid":1000,"mapped_gid":1000}` + w := do(t, h, "POST", "/netstorage/add", "A", body) + if w.Code != http.StatusOK { + t.Fatalf("add NFS: got %d want 200 (%s)", w.Code, w.Body.String()) + } + if len(n.ensured) != 1 { + t.Fatalf("EnsureNetworkMount not called once: %v", n.ensured) + } + got := n.ensured[0] + if got.Name != "media" || got.Protocol != storage.ProtocolNFS || got.Server != "10.0.0.5" || got.Export != "/srv/media" { + t.Fatalf("spec mismatch: %+v", got) + } + if got.HostUID() != 101000 { + t.Errorf("host uid = %d want 101000 (the +100000 recipe)", got.HostUID()) + } +} + +func TestNetStorage_AddSMB_WritesCreds0600(t *testing.T) { + credsDir := t.TempDir() + n := &fakeNetOps{} + h := newNetServer(t, n, credsDir).Handler() + body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}` + w := do(t, h, "POST", "/netstorage/add", "A", body) + if w.Code != http.StatusOK { + t.Fatalf("add SMB: got %d want 200 (%s)", w.Code, w.Body.String()) + } + if len(n.ensured) != 1 || n.ensured[0].CredsRef == "" { + t.Fatalf("SMB spec must carry a creds ref: %+v", n.ensured) + } + credsPath := filepath.Join(credsDir, "vids.cred") + info, err := os.Stat(credsPath) + if err != nil { + t.Fatalf("creds file not written: %v", err) + } + // 0600 is enforced on Linux (the production OS); Windows does not honor Unix perms, so the perm + // assertion runs on the Linux build server where it matters. + if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 { + t.Errorf("creds file is group/other-readable (mode %v) — must be 0600", info.Mode().Perm()) + } + data, _ := os.ReadFile(credsPath) + if !strings.Contains(string(data), "username=u") || !strings.Contains(string(data), "password=p") { + t.Errorf("creds file content wrong: %q", data) + } + // The secret must NOT appear in the response body. + if strings.Contains(w.Body.String(), "\"p\"") || strings.Contains(w.Body.String(), "password") { + t.Errorf("response leaked credentials: %s", w.Body.String()) + } +} + +func TestNetStorage_AddSMB_MissingCreds_Refused(t *testing.T) { + n := &fakeNetOps{} + h := newNetServer(t, n, t.TempDir()).Handler() + body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000}` + w := do(t, h, "POST", "/netstorage/add", "A", body) + if w.Code != http.StatusBadRequest { + t.Fatalf("SMB without creds: got %d want 400", w.Code) + } + if len(n.ensured) != 0 { + t.Fatal("EnsureNetworkMount must not be called when creds are missing") + } +} + +// TestNetStorage_RoleGate_NonUserDataRefused: a mount root outside the user-data namespace is refused +// (403) and the mount surface is never touched — network storage is bulk-userdata ONLY. +func TestNetStorage_RoleGate_NonUserDataRefused(t *testing.T) { + n := &fakeNetOps{} + srv := newNetServer(t, n, t.TempDir()) + srv.netMountRoot = "/srv/system" // a non-user-data path + h := srv.Handler() + body := `{"name":"media","protocol":"nfs","server":"10.0.0.5","export":"/srv/media","mapped_uid":1000,"mapped_gid":1000}` + w := do(t, h, "POST", "/netstorage/add", "A", body) + if w.Code != http.StatusForbidden { + t.Fatalf("non-user-data path: got %d want 403 (%s)", w.Code, w.Body.String()) + } + if len(n.ensured) != 0 { + t.Fatal("a role-gated request must not reach EnsureNetworkMount") + } +} + +func TestNetStorage_List(t *testing.T) { + n := &fakeNetOps{list: []storage.NetworkMountStatus{ + {Name: "media", Protocol: "nfs", Server: "10.0.0.5", Where: "/mnt/felhom-drives/media", Configured: true, Mounted: true, Reachable: true, Health: storage.NetHealthOK}, + }} + h := newNetServer(t, n, t.TempDir()).Handler() + w := do(t, h, "GET", "/netstorage", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("list: got %d want 200 (%s)", w.Code, w.Body.String()) + } + var resp struct { + Data struct { + NetworkMounts []storage.NetworkMountStatus `json:"network_mounts"` + } `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if len(resp.Data.NetworkMounts) != 1 || resp.Data.NetworkMounts[0].Health != storage.NetHealthOK { + t.Fatalf("list payload wrong: %+v", resp.Data.NetworkMounts) + } +} + +func TestNetStorage_Remove(t *testing.T) { + credsDir := t.TempDir() + // Pre-stage a creds file to prove removal cleans it up. + credsPath := filepath.Join(credsDir, "media.cred") + if err := os.WriteFile(credsPath, []byte("username=u\npassword=p\n"), 0o600); err != nil { + t.Fatal(err) + } + n := &fakeNetOps{} + h := newNetServer(t, n, credsDir).Handler() + w := do(t, h, "POST", "/netstorage/remove", "A", `{"name":"media"}`) + if w.Code != http.StatusOK { + t.Fatalf("remove: got %d want 200 (%s)", w.Code, w.Body.String()) + } + if len(n.removed) != 1 || n.removed[0] != "media" { + t.Fatalf("RemoveNetworkMount not called: %v", n.removed) + } + if _, err := os.Stat(credsPath); !os.IsNotExist(err) { + t.Errorf("creds file should be removed, stat err = %v", err) + } +} + +func TestNetStorage_NotConfigured(t *testing.T) { + h := newNetServer(t, nil, "").Handler() // NetStorage nil + for _, tc := range []struct{ method, path, body string }{ + {"POST", "/netstorage/add", `{"name":"media","protocol":"nfs","server":"x","export":"/y","mapped_uid":1000,"mapped_gid":1000}`}, + {"GET", "/netstorage", ""}, + {"POST", "/netstorage/remove", `{"name":"media"}`}, + } { + w := do(t, h, tc.method, tc.path, "A", tc.body) + if w.Code != http.StatusServiceUnavailable { + t.Errorf("%s %s with no NetStorage: got %d want 503", tc.method, tc.path, w.Code) + } + } +} + +func TestNetStorage_RequiresAuth(t *testing.T) { + h := newNetServer(t, &fakeNetOps{}, t.TempDir()).Handler() + w := do(t, h, "GET", "/netstorage", "", "") + if w.Code != http.StatusUnauthorized { + t.Fatalf("no token: got %d want 401", w.Code) + } +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index 6d6c213..5003c35 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -84,6 +84,12 @@ type Options struct { // GuestAttach binds an enrolled user-data drive's felhom-data namespace into the guest (slice 10 // P2, Model A). OPTIONAL — when nil, POST /disks/guest-attach reports "not configured". GuestAttach GuestAttacher + // NetStorage is the privileged network-mount (NAS) surface (Part A1). OPTIONAL — when nil, the + // /netstorage endpoints report "not configured". Satisfied by *storage.SudoHostOps. + NetStorage NetworkStorageOps + // SmbCredsDir is where the agent writes the 0600 SMB credentials files (out-of-band). "" → + // /var/lib/felhom-agent/smb-creds. + SmbCredsDir string // ControllerSwap runs guest commands (pct exec) for the agentic controller-update swap (Phase 1). // OPTIONAL — when nil, POST /controller/swap reports "not configured". Satisfied by *GuestBinder. ControllerSwap GuestExecutor @@ -158,6 +164,9 @@ type Server struct { diskGate StorageGate // slice 8C (optional) guestList GuestLister // slice 8C (optional) guestAttach GuestAttacher // slice 10 P2 (optional) + netStorage NetworkStorageOps // Part A1: NAS network mounts (optional) + netMountRoot string // the user-data namespace root for the network-mount role gate + smbCredsDir string // where SMB creds files are written (out-of-band, 0600) intent IntentRecorder // slice 10 P3 (optional) guestBinds *GuestBindStore // F9 startup bind re-assert record (optional) formatJobs *FormatJobStore // F20-BUG3 detached-format job record (optional) @@ -225,6 +234,9 @@ func NewServer(o Options) (*Server, error) { diskGate: o.DiskGate, guestList: o.Guests2, guestAttach: o.GuestAttach, + netStorage: o.NetStorage, + netMountRoot: storage.NetworkMountRoot, + smbCredsDir: o.SmbCredsDir, intent: o.Intent, guestBinds: o.GuestBinds, formatJobs: o.FormatJobs, @@ -268,6 +280,12 @@ func (s *Server) Handler() http.Handler { // Guest reboot (slice 10 P2 activation): user-triggered restart to activate pending drive binds. mux.HandleFunc("POST /guest/reboot", s.withGuest(s.handleGuestReboot)) + // Network storage (NAS) — Part A1: mount/list/remove a bulk-media NAS share host-side (automount + // idle-unmount; +100000 uid recipe). A distinct class from a drive — no enroll/eject/wipe. + mux.HandleFunc("POST /netstorage/add", s.withGuest(s.handleNetStorageAdd)) + mux.HandleFunc("GET /netstorage", s.withGuest(s.handleNetStorageList)) + mux.HandleFunc("POST /netstorage/remove", s.withGuest(s.handleNetStorageRemove)) + // agentic controller update (Phase 1): in-guest image swap + rollback, owned by the agent. mux.HandleFunc("POST /controller/swap", s.withGuest(s.handleControllerSwap)) mux.HandleFunc("GET /controller/swap/status", s.withGuest(s.handleControllerSwapStatus)) diff --git a/internal/storage/mountunit.go b/internal/storage/mountunit.go index 279e18d..699dc95 100644 --- a/internal/storage/mountunit.go +++ b/internal/storage/mountunit.go @@ -16,6 +16,13 @@ func parseFelhomMountUnit(content string) (MountSpec, bool) { if !strings.Contains(content, felhomUnitMarker) { return MountSpec{}, false } + // GUARD (network-storage bypass, Scenario D): a NAS network mount carries the netUnitMarker, which + // also contains felhomUnitMarker as a substring. It is NOT a drive — no durable-id, no device — and + // MUST NOT be picked up by the host-reboot drive re-assert. Refuse it explicitly here so even a + // (contrived) network unit shaped like a by-uuid drive unit can never be classified as a drive. + if strings.Contains(content, netUnitMarker) { + return MountSpec{}, false + } var spec MountSpec for _, line := range strings.Split(content, "\n") { line = strings.TrimSpace(line) diff --git a/internal/storage/netmount.go b/internal/storage/netmount.go new file mode 100644 index 0000000..3994355 --- /dev/null +++ b/internal/storage/netmount.go @@ -0,0 +1,511 @@ +package storage + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// Network storage (NAS) — the bulk-media class (Part A1). A NAS share is mounted HOST-SIDE under +// /mnt/felhom-drives/ via a systemd .automount (on-demand + idle-unmount) + .mount pair; it +// propagates into the guest for free through the existing shared mp8 bind. This is a DISTINCT class +// from a physical drive: it carries NO durable-id, never enters the IntentStore, is never SMART-probed, +// and bypasses the enroll/eject/decommission/wipe/migrate machinery entirely (SPIKE-nas-storage Q7). +// +// The whole file is the validated, locked recipe from SPIKE-nas-storage-2026-06-29.md — match it, do +// not re-derive: NFS preferred (vers=4.1,soft,timeo=50,retrans=2,noatime), SMB fallback, the +100000 +// uid rule (container uid/gid N = host N+100000), automount idle-unmount for failure isolation. + +const ( + // NetworkMountRoot is the ONLY place a network mount may live — under the existing shared mp8 + // bind parent, so it propagates into the guest with no new mountpoint and no restart. The role + // gate (NetworkMountRole) confines every network mount to this namespace. + NetworkMountRoot = "/mnt/felhom-drives" + + // lxcUIDOffset is the unprivileged-LXC id map base: a container uid/gid N appears on the host as + // N+100000 (verified against the live felhom-usb userdata). The NAS must present/own/squash files + // as container_id+100000 so the guest sees its native id and reads+writes (SPIKE Q5). This is the + // whole +100000 recipe; a naive +0 (anonuid=1000 / uid=1000) lands as nobody:nogroup → not writable. + lxcUIDOffset = 100000 + + // netUnitMarker headers every network-storage unit. It is the explicit guard that keeps a network + // mount OUT of the drive machinery: parseFelhomMountUnit refuses any unit carrying it, so the + // host-reboot drive re-assert (ReassertEnrolledMounts) never touches a NAS mount (Scenario D). + netUnitMarker = "Managed by felhom-agent (network storage)" + + // defaultIdleTimeoutSec is the automount TimeoutIdleSec: with no app reading, the share auto-unmounts, + // so an idle NAS reboot is a non-event and the stale-mount window is minimised (SPIKE recommendation). + defaultIdleTimeoutSec = 60 + + // netReachTimeout bounds the per-share liveness endpoint dial so a black-holed NAS cannot wedge a + // list call (the mount itself may be EIO/D-state; we probe the endpoint, never stat the mount). + netReachTimeout = 2 * time.Second +) + +// NetworkProtocol is the wire protocol of a network mount. +type NetworkProtocol string + +const ( + ProtocolNFS NetworkProtocol = "nfs" + ProtocolSMB NetworkProtocol = "smb" +) + +// NetworkMountSpec describes one NAS share to mount host-side. It is NOT a drive: no durable-id, no +// device, no role lifecycle. MappedUID/MappedGID are the CONTAINER ids (e.g. 1000); the agent applies +// the +100000 offset where it matters (the SMB client mount). CredsRef points at the 0600 out-of-band +// SMB credentials file (empty for NFS) — never the credentials themselves. +type NetworkMountSpec struct { + Name string // share name → mountpoint /mnt/felhom-drives/; a single safe path segment + Protocol NetworkProtocol // nfs | smb + Server string // NAS host or IP + Export string // NFS export path (/srv/media) or SMB share name (media) + MappedUID int // CONTAINER uid the media app runs as (host = +100000) + MappedGID int // CONTAINER gid + CredsRef string // SMB only: absolute path to the 0600 credentials file (out-of-band) + IdleTimeoutSec int // automount idle-unmount window; 0 → defaultIdleTimeoutSec +} + +// NetworkMountStatus is the per-share liveness surface (SPIKE Q7 health model): is it configured, +// currently mounted, and is the NAS endpoint reachable. Health degrades to "unreachable" for the +// affected share ONLY — it never folds into the box's overall health. +type NetworkMountStatus struct { + Name string `json:"name"` + Protocol string `json:"protocol"` + Server string `json:"server"` + Export string `json:"export"` + Where string `json:"where"` + Configured bool `json:"configured"` // the automount unit is installed + Mounted bool `json:"mounted"` // currently mounted (false when idle-unmounted — normal, not a fault) + Reachable bool `json:"reachable"` // the NAS endpoint is TCP-reachable + Health string `json:"health"` // ok | idle | unreachable +} + +// Network mount health vocabulary. +const ( + NetHealthOK = "ok" // reachable + mounted: serving + NetHealthIdle = "idle" // reachable + not mounted: automount idle-unmounted (benign) + NetHealthUnreachable = "unreachable" // endpoint not reachable: the affected share is degraded +) + +var ( + // reShareName: a single safe path segment for the share name (becomes the mountpoint dir + unit name). + reShareName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + // reNetServer: a hostname or IPv4/IPv6 literal — no metacharacters, no '/', no ':' path tricks. + reNetServer = regexp.MustCompile(`^[A-Za-z0-9._:-]+$`) +) + +const maxShareNameLen = 64 + +// Where returns the mountpoint for this spec under the network-mount root. +func (s NetworkMountSpec) Where() string { + return NetworkMountRoot + "/" + s.Name +} + +// HostUID/HostGID apply the +100000 LXC offset — the host id the share must own/squash files as. +func (s NetworkMountSpec) HostUID() int { return s.MappedUID + lxcUIDOffset } +func (s NetworkMountSpec) HostGID() int { return s.MappedGID + lxcUIDOffset } + +// idleTimeout returns the configured idle-unmount window or the default. +func (s NetworkMountSpec) idleTimeout() int { + if s.IdleTimeoutSec > 0 { + return s.IdleTimeoutSec + } + return defaultIdleTimeoutSec +} + +// ValidateNetworkMountSpec is the security boundary for the network-mount surface — every value that +// reaches a systemd unit (and thus a root-triggered mount) is checked here before any unit is rendered. +// Mirrors validate.go's discipline: strict charset/length, no metacharacters, no traversal. +func ValidateNetworkMountSpec(s NetworkMountSpec) error { + if s.Name == "" || len(s.Name) > maxShareNameLen || !reShareName.MatchString(s.Name) || s.Name == "." || s.Name == ".." { + return fmt.Errorf("netmount: invalid share name %q (want a single safe segment)", s.Name) + } + switch s.Protocol { + case ProtocolNFS, ProtocolSMB: + default: + return fmt.Errorf("netmount: unsupported protocol %q (want nfs|smb)", s.Protocol) + } + if s.Server == "" || len(s.Server) > 255 || !reNetServer.MatchString(s.Server) { + return fmt.Errorf("netmount: invalid server %q", s.Server) + } + if err := validateExport(s.Protocol, s.Export); err != nil { + return err + } + if s.MappedUID < 0 || s.MappedUID > 60000 || s.MappedGID < 0 || s.MappedGID > 60000 { + return fmt.Errorf("netmount: mapped uid/gid out of range (uid=%d gid=%d; want 0..60000)", s.MappedUID, s.MappedGID) + } + if s.Protocol == ProtocolSMB { + if s.CredsRef == "" { + return fmt.Errorf("netmount: smb requires a credentials file reference") + } + if err := ValidateMountPath(s.CredsRef); err != nil { + return fmt.Errorf("netmount: invalid credentials path: %w", err) + } + } + // The mountpoint must validate AND must land under the network-mount root (the role gate's namespace). + if err := ValidateMountPath(s.Where()); err != nil { + return err + } + return nil +} + +// validateExport checks the NFS export path / SMB share name. NFS export must be an absolute, safe path; +// the SMB share name a single safe segment. +func validateExport(proto NetworkProtocol, export string) error { + if export == "" { + return fmt.Errorf("netmount: empty export/share") + } + switch proto { + case ProtocolNFS: + if export[0] != '/' { + return fmt.Errorf("netmount: nfs export must be an absolute path, got %q", export) + } + if err := ValidateMountPath(export); err != nil { + return fmt.Errorf("netmount: invalid nfs export: %w", err) + } + case ProtocolSMB: + if len(export) > maxShareNameLen || !reShareName.MatchString(export) { + return fmt.Errorf("netmount: invalid smb share name %q", export) + } + } + return nil +} + +// NetworkMountRole gates network storage to the bulk-userdata namespace (SPIKE Q7: a NAS is selectable +// as a media app's data path but DISQUALIFIED as a system/backup/DB target). A network mount is +// user-data ONLY when it lands under NetworkMountRoot (the controller's bind blast radius); any other +// target is the most-protected role (system) → the caller refuses it. Pure → unit-tested. +func NetworkMountRole(where string) DeviceRole { + clean := cleanMountPath(where) + if clean == NetworkMountRoot || strings.HasPrefix(clean, NetworkMountRoot+"/") { + return RoleUserData + } + return RoleSystem +} + +// mountSource builds the systemd What= for the spec: server:/export (NFS) or //server/share (SMB). +func (s NetworkMountSpec) mountSource() string { + if s.Protocol == ProtocolSMB { + return "//" + s.Server + "/" + s.Export + } + return s.Server + ":" + s.Export +} + +// fsType maps the protocol to the kernel filesystem type for the .mount unit. +func (s NetworkMountSpec) fsType() string { + if s.Protocol == ProtocolSMB { + return "cifs" + } + return "nfs4" // vers=4.1 → nfs4 (avoids the rpcbind/lock-manager surface of v3) +} + +// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5): +// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev — soft is the failure-isolation knob +// (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not the +// client mount, so no uid appears here. +// - SMB: vers=3.0,credentials=,uid=<+100000>,gid=<+100000>,forceuid,forcegid,file_mode=0664, +// dir_mode=0775,_netdev — modes are PLAIN octal (not setgid 2775); the client forces the +// guest-visible owner to the mapped id so the container reads+writes. +// +// Every interpolated value is pre-validated by ValidateNetworkMountSpec, so the string carries no +// newline / no extra directive. NEVER a default `hard` NFS mount (it wedges) — soft is mandatory. +func (s NetworkMountSpec) mountOptions() string { + if s.Protocol == ProtocolSMB { + return strings.Join([]string{ + "vers=3.0", + "credentials=" + s.CredsRef, + "uid=" + strconv.Itoa(s.HostUID()), + "gid=" + strconv.Itoa(s.HostGID()), + "forceuid", + "forcegid", + "file_mode=0664", + "dir_mode=0775", + "_netdev", + }, ",") + } + return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev" +} + +// renderNetworkMountUnit builds the .mount unit (triggered by the .automount; deliberately NO [Install] +// — we enable the .automount, not this). Marked with netUnitMarker so the drive machinery skips it. +func renderNetworkMountUnit(s NetworkMountSpec) string { + var b strings.Builder + b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n") + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "Description=Felhom network storage %s (%s)\n", sanitizeDesc(s.Name), s.Protocol) + b.WriteString("After=network-online.target\n") + b.WriteString("Wants=network-online.target\n") + b.WriteString("\n[Mount]\n") + fmt.Fprintf(&b, "What=%s\n", s.mountSource()) + fmt.Fprintf(&b, "Where=%s\n", s.Where()) + fmt.Fprintf(&b, "Type=%s\n", s.fsType()) + fmt.Fprintf(&b, "Options=%s\n", s.mountOptions()) + return b.String() +} + +// renderNetworkAutomountUnit builds the .automount unit (on-demand + idle-unmount). Enabling THIS is +// what realises the on-demand mount; the idle timeout makes an idle NAS reboot a non-event. +func renderNetworkAutomountUnit(s NetworkMountSpec) string { + var b strings.Builder + b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n") + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "Description=Felhom network storage automount %s (%s)\n", sanitizeDesc(s.Name), s.Protocol) + b.WriteString("After=network-online.target\n") + b.WriteString("Wants=network-online.target\n") + b.WriteString("\n[Automount]\n") + fmt.Fprintf(&b, "Where=%s\n", s.Where()) + fmt.Fprintf(&b, "TimeoutIdleSec=%d\n", s.idleTimeout()) + b.WriteString("\n[Install]\n") + b.WriteString("WantedBy=multi-user.target\n") + return b.String() +} + +// parseNetworkMountUnit is the inverse of renderNetworkMountUnit for the fields the liveness list needs. +// ok=false unless the content is a felhom network-storage .mount unit (the netUnitMarker + a parseable +// network What=). Pure → unit-tested. +func parseNetworkMountUnit(content string) (proto, server, export, where string, ok bool) { + if !strings.Contains(content, netUnitMarker) { + return "", "", "", "", false + } + var what, fstype string + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "What="): + what = strings.TrimPrefix(line, "What=") + case strings.HasPrefix(line, "Where="): + where = strings.TrimPrefix(line, "Where=") + case strings.HasPrefix(line, "Type="): + fstype = strings.TrimPrefix(line, "Type=") + } + } + if where == "" || what == "" { + return "", "", "", "", false + } + switch fstype { + case "cifs": + proto = string(ProtocolSMB) + // //server/share + rest := strings.TrimPrefix(what, "//") + if i := strings.IndexByte(rest, '/'); i > 0 { + server, export = rest[:i], rest[i+1:] + } + case "nfs4", "nfs": + proto = string(ProtocolNFS) + if i := strings.IndexByte(what, ':'); i > 0 { + server, export = what[:i], what[i+1:] + } + default: + return "", "", "", "", false + } + if server == "" { + return "", "", "", "", false + } + return proto, server, export, where, true +} + +// networkHealth derives the per-share health string from reachability + mount state (SPIKE Q7). +func networkHealth(reachable, mounted bool) string { + switch { + case !reachable: + return NetHealthUnreachable + case mounted: + return NetHealthOK + default: + return NetHealthIdle // reachable but idle-unmounted — normal automount steady state + } +} + +// netEndpoint returns the host:port the liveness probe dials for a protocol (NFS 2049, SMB 445). +func netEndpoint(proto, server string) string { + port := "2049" + if proto == string(ProtocolSMB) { + port = "445" + } + return netJoin(server, port) +} + +// ---- SudoHostOps: the privileged network-mount surface -------------------------------------------- + +// EnsureNetworkMount stages + installs the .mount and .automount units and enables the AUTOMOUNT (not +// the mount): the share then mounts on first access and idle-unmounts when quiet. Idempotent. It does +// NOT register a durable-id, NOT enter the IntentStore, NOT SMART-probe — a NAS is not a drive. +func (h *SudoHostOps) EnsureNetworkMount(ctx context.Context, spec NetworkMountSpec) error { + if err := ValidateNetworkMountSpec(spec); err != nil { + return err + } + // Defense in depth: never realise a network mount outside the user-data namespace. + if NetworkMountRole(spec.Where()) != RoleUserData { + return fmt.Errorf("netmount: refusing to mount outside the user-data namespace: %s", spec.Where()) + } + mountUnit, err := UnitNameForMount(spec.Where()) + if err != nil { + return err + } + automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount" + + // Ensure the mountpoint exists (systemd automount also creates it, but be explicit — the parent is + // the shared bind root). Reuse the intermediary mkdir grant. + if err := h.run(ctx, "/usr/bin/mkdir", "-p", spec.Where()); err != nil { + return fmt.Errorf("netmount: mkdir %s: %w", spec.Where(), err) + } + + if err := h.installUnit(ctx, mountUnit, renderNetworkMountUnit(spec)); err != nil { + return err + } + if err := h.installUnit(ctx, automountUnit, renderNetworkAutomountUnit(spec)); err != nil { + return err + } + if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil { + return fmt.Errorf("netmount: daemon-reload: %w", err) + } + // Enable + start the AUTOMOUNT (creates the autofs trigger; idempotent). + if err := h.run(ctx, h.bins.Systemctl, "enable", "--now", "--", automountUnit); err != nil { + return fmt.Errorf("netmount: enabling automount %s: %w", automountUnit, err) + } + h.logger.Info("netmount: ensured network mount", "name", spec.Name, "proto", spec.Protocol, + "server", spec.Server, "export", spec.Export, "where", spec.Where(), "host_uid", spec.HostUID()) + return nil +} + +// installUnit stages an agent-owned unit file then root-installs it into the unit dir (atomic, fixed +// mode/owner) — the same staging pattern as EnsureMount. +func (h *SudoHostOps) installUnit(ctx context.Context, unitName, content string) error { + if err := os.MkdirAll(h.stageDir, 0o700); err != nil { + return fmt.Errorf("netmount: staging dir: %w", err) + } + stagePath := filepath.Join(h.stageDir, unitName) + if err := os.WriteFile(stagePath, []byte(content), 0o644); err != nil { + return fmt.Errorf("netmount: staging unit %s: %w", unitName, err) + } + dest := filepath.Join(h.unitDir, unitName) + if err := h.run(ctx, h.bins.Install, "-o", "root", "-g", "root", "-m", "0644", "--", stagePath, dest); err != nil { + return fmt.Errorf("netmount: installing unit %s: %w", unitName, err) + } + return nil +} + +// RemoveNetworkMount stops + disables the automount, stops the mount, and removes both unit files (and +// the staged copies). It NEVER routes through the drive eject/decommission path — a NAS has no device +// lifecycle. Best-effort on the per-step stop/disable (a not-loaded unit is fine); the file removal is +// the authoritative "gone" signal. +func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error { + if name == "" || len(name) > maxShareNameLen || !reShareName.MatchString(name) || name == "." || name == ".." { + return fmt.Errorf("netmount: invalid share name %q", name) + } + where := NetworkMountRoot + "/" + name + mountUnit, err := UnitNameForMount(where) + if err != nil { + return err + } + automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount" + + // Stop the automount first (so it can't re-trigger the mount), then the mount. Tolerate "not loaded". + _ = h.run(ctx, h.bins.Systemctl, "stop", "--", automountUnit) + _ = h.run(ctx, h.bins.Systemctl, "disable", "--", automountUnit) + _ = h.run(ctx, h.bins.Systemctl, "stop", "--", mountUnit) + + destAuto := filepath.Join(h.unitDir, automountUnit) + destMount := filepath.Join(h.unitDir, mountUnit) + if err := h.run(ctx, "/usr/bin/rm", "-f", destAuto); err != nil { + return fmt.Errorf("netmount: removing %s: %w", automountUnit, err) + } + if err := h.run(ctx, "/usr/bin/rm", "-f", destMount); err != nil { + return fmt.Errorf("netmount: removing %s: %w", mountUnit, err) + } + _ = os.Remove(filepath.Join(h.stageDir, automountUnit)) + _ = os.Remove(filepath.Join(h.stageDir, mountUnit)) + if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil { + return fmt.Errorf("netmount: daemon-reload: %w", err) + } + h.logger.Info("netmount: removed network mount", "name", name, "where", where) + return nil +} + +// ListNetworkMounts enumerates the installed network-storage units and reports per-share liveness. It +// reads the (world-readable) unit dir + /proc/mounts and TCP-probes each NAS endpoint with a short +// timeout — it NEVER stat()s the (possibly EIO/D-state) mountpoint, so a black-holed NAS cannot wedge +// this call. Best-effort: an unreadable unit dir yields an empty list. +func (h *SudoHostOps) ListNetworkMounts(ctx context.Context) ([]NetworkMountStatus, error) { + entries, err := os.ReadDir(h.unitDir) + if err != nil { + return nil, fmt.Errorf("netmount: reading unit dir: %w", err) + } + mounts := h.mountedFSTypes() + var out []NetworkMountStatus + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".mount") { + continue // the .mount unit carries the What/Type; the .automount mirrors Where only + } + data, rerr := os.ReadFile(filepath.Join(h.unitDir, e.Name())) + if rerr != nil { + continue + } + proto, server, export, where, ok := parseNetworkMountUnit(string(data)) + if !ok { + continue // not one of ours (or a drive by-uuid mount) + } + st := NetworkMountStatus{ + Name: strings.TrimPrefix(where, NetworkMountRoot+"/"), + Protocol: proto, + Server: server, + Export: export, + Where: where, + Configured: true, + Mounted: isNetworkMounted(mounts[where]), + Reachable: endpointReachable(netEndpoint(proto, server)), + } + st.Health = networkHealth(st.Reachable, st.Mounted) + out = append(out, st) + } + return out, nil +} + +// mountedFSTypes maps each active mountpoint to its filesystem type (from /proc/mounts: field 2 = where, +// field 3 = fstype). Best-effort. +func (h *SudoHostOps) mountedFSTypes() map[string]string { + out := map[string]string{} + data, err := os.ReadFile("/proc/mounts") + if err != nil { + return out + } + for _, line := range strings.Split(string(data), "\n") { + f := strings.Fields(line) + if len(f) >= 3 { + out[f[1]] = f[2] + } + } + return out +} + +// isNetworkMounted reports whether the recorded fstype at a mountpoint is a real network mount (not the +// autofs trigger). An idle automount shows fstype "autofs" (or nothing) → not mounted; an active mount +// shows nfs4/nfs/cifs. +func isNetworkMounted(fstype string) bool { + switch fstype { + case "nfs", "nfs4", "cifs": + return true + default: + return false + } +} + +// endpointReachable TCP-dials a NAS endpoint with a short timeout (the liveness probe that never touches +// the mount). "" endpoint → not reachable. +func endpointReachable(endpoint string) bool { + if endpoint == "" { + return false + } + conn, err := net.DialTimeout("tcp", endpoint, netReachTimeout) + if err != nil { + return false + } + _ = conn.Close() + return true +} diff --git a/internal/storage/netmount_test.go b/internal/storage/netmount_test.go new file mode 100644 index 0000000..c1552af --- /dev/null +++ b/internal/storage/netmount_test.go @@ -0,0 +1,334 @@ +package storage + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// --- Unit rendering: the exact, locked SPIKE option sets + the +100000 recipe ----------------------- + +func TestNetworkMount_NFSUnitRendering(t *testing.T) { + spec := NetworkMountSpec{ + Name: "media", Protocol: ProtocolNFS, Server: "192.168.0.180", Export: "/srv/nas-sim/media", + MappedUID: 1000, MappedGID: 1000, + } + if err := ValidateNetworkMountSpec(spec); err != nil { + t.Fatalf("valid NFS spec rejected: %v", err) + } + mu := renderNetworkMountUnit(spec) + + wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev" + for _, want := range []string{ + netUnitMarker, + "What=192.168.0.180:/srv/nas-sim/media", + "Where=/mnt/felhom-drives/media", + "Type=nfs4", + wantOpts, + } { + if !strings.Contains(mu, want) { + t.Errorf("NFS .mount missing %q:\n%s", want, mu) + } + } + // soft is the failure-isolation knob; a default hard mount wedges — it must NEVER appear. + if strings.Contains(mu, "hard") { + t.Errorf("NFS mount must not be hard:\n%s", mu) + } + // NFS uid mapping is the EXPORT's job (anonuid=101000) — the client mount carries no uid/gid. + if strings.Contains(mu, "uid=") || strings.Contains(mu, "anonuid") { + t.Errorf("NFS client mount must not carry uid options (server-side squash):\n%s", mu) + } + + au := renderNetworkAutomountUnit(spec) + for _, want := range []string{ + netUnitMarker, + "Where=/mnt/felhom-drives/media", + "[Automount]", + "TimeoutIdleSec=60", + "WantedBy=multi-user.target", + } { + if !strings.Contains(au, want) { + t.Errorf("NFS .automount missing %q:\n%s", want, au) + } + } +} + +// TestNetworkMount_SMBUnitRendering_Plus100000 is the headline +100000 companion: a container uid/gid of +// 1000 MUST render the SMB client mount with uid=101000/gid=101000 (host = container+100000). A naive +// +0 implementation (uid=1000) would FAIL here — and lands as nobody:nogroup in the guest (not writable). +func TestNetworkMount_SMBUnitRendering_Plus100000(t *testing.T) { + spec := NetworkMountSpec{ + Name: "media", Protocol: ProtocolSMB, Server: "nas.local", Export: "media", + MappedUID: 1000, MappedGID: 1000, CredsRef: "/var/lib/felhom-agent/smb-creds/media.cred", + } + if err := ValidateNetworkMountSpec(spec); err != nil { + t.Fatalf("valid SMB spec rejected: %v", err) + } + mu := renderNetworkMountUnit(spec) + + wantOpts := "Options=vers=3.0,credentials=/var/lib/felhom-agent/smb-creds/media.cred,uid=101000,gid=101000,forceuid,forcegid,file_mode=0664,dir_mode=0775,_netdev" + for _, want := range []string{ + netUnitMarker, + "What=//nas.local/media", + "Where=/mnt/felhom-drives/media", + "Type=cifs", + wantOpts, + } { + if !strings.Contains(mu, want) { + t.Errorf("SMB .mount missing %q:\n%s", want, mu) + } + } + // THE companion red-proof: the +100000 offset must be applied; a +0 impl emits uid=1000. + if strings.Contains(mu, "uid=1000,") || strings.Contains(mu, "gid=1000,") { + t.Errorf("SMB mount used the raw container id, not +100000 (the documented non-writable trap):\n%s", mu) + } + // Modes must be PLAIN octal (0664/0775), never the setgid 2775 the drive userdata uses. + if strings.Contains(mu, "2775") { + t.Errorf("SMB dir_mode must be plain octal 0775, not setgid 2775:\n%s", mu) + } +} + +func TestNetworkMount_HostOffset(t *testing.T) { + s := NetworkMountSpec{MappedUID: 1000, MappedGID: 1000} + if s.HostUID() != 101000 || s.HostGID() != 101000 { + t.Fatalf("HostUID/HostGID = %d/%d, want 101000/101000", s.HostUID(), s.HostGID()) + } +} + +// --- Validation ------------------------------------------------------------------------------------- + +func TestValidateNetworkMountSpec(t *testing.T) { + base := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000} + good := func(mut func(*NetworkMountSpec)) NetworkMountSpec { s := base; mut(&s); return s } + + bad := []struct { + name string + spec NetworkMountSpec + }{ + {"empty name", good(func(s *NetworkMountSpec) { s.Name = "" })}, + {"name traversal", good(func(s *NetworkMountSpec) { s.Name = ".." })}, + {"name with slash", good(func(s *NetworkMountSpec) { s.Name = "a/b" })}, + {"name with space", good(func(s *NetworkMountSpec) { s.Name = "a b" })}, + {"bad protocol", good(func(s *NetworkMountSpec) { s.Protocol = "afp" })}, + {"server metachar", good(func(s *NetworkMountSpec) { s.Server = "a;rm -rf" })}, + {"empty server", good(func(s *NetworkMountSpec) { s.Server = "" })}, + {"nfs relative export", good(func(s *NetworkMountSpec) { s.Export = "srv/media" })}, + {"nfs export traversal", good(func(s *NetworkMountSpec) { s.Export = "/srv/../etc" })}, + {"uid out of range", good(func(s *NetworkMountSpec) { s.MappedUID = 70000 })}, + {"negative gid", good(func(s *NetworkMountSpec) { s.MappedGID = -1 })}, + {"smb without creds", good(func(s *NetworkMountSpec) { s.Protocol = ProtocolSMB; s.Export = "media"; s.CredsRef = "" })}, + {"smb bad share name", good(func(s *NetworkMountSpec) { s.Protocol = ProtocolSMB; s.Export = "a/b"; s.CredsRef = "/x/y.cred" })}, + } + for _, c := range bad { + if err := ValidateNetworkMountSpec(c.spec); err == nil { + t.Errorf("%s: expected rejection, got nil", c.name) + } + } + + // Good specs. + if err := ValidateNetworkMountSpec(base); err != nil { + t.Errorf("valid NFS spec rejected: %v", err) + } + smb := good(func(s *NetworkMountSpec) { s.Protocol = ProtocolSMB; s.Export = "media"; s.CredsRef = "/var/lib/felhom-agent/smb-creds/media.cred" }) + if err := ValidateNetworkMountSpec(smb); err != nil { + t.Errorf("valid SMB spec rejected: %v", err) + } +} + +// --- Role gate: bulk-userdata namespace only -------------------------------------------------------- + +func TestNetworkMountRole(t *testing.T) { + userdata := []string{"/mnt/felhom-drives/media", "/mnt/felhom-drives/photos", NetworkMountRoot} + for _, p := range userdata { + if NetworkMountRole(p) != RoleUserData { + t.Errorf("%s should be user-data", p) + } + } + system := []string{"/etc/passwd", "/srv/system/x", "/mnt/felhom-drivesX/y", "/var/lib/felhom-agent"} + for _, p := range system { + if NetworkMountRole(p) != RoleSystem { + t.Errorf("%s should be system (refused)", p) + } + } +} + +// --- Liveness parse + health ------------------------------------------------------------------------ + +func TestParseNetworkMountUnit_RoundTrip(t *testing.T) { + nfs := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000} + proto, server, export, where, ok := parseNetworkMountUnit(renderNetworkMountUnit(nfs)) + if !ok || proto != "nfs" || server != "10.0.0.5" || export != "/srv/media" || where != "/mnt/felhom-drives/media" { + t.Errorf("NFS parse = %q %q %q %q ok=%v", proto, server, export, where, ok) + } + smb := NetworkMountSpec{Name: "vids", Protocol: ProtocolSMB, Server: "nas", Export: "vids", MappedUID: 1000, MappedGID: 1000, CredsRef: "/x/y.cred"} + proto, server, export, where, ok = parseNetworkMountUnit(renderNetworkMountUnit(smb)) + if !ok || proto != "smb" || server != "nas" || export != "vids" || where != "/mnt/felhom-drives/vids" { + t.Errorf("SMB parse = %q %q %q %q ok=%v", proto, server, export, where, ok) + } + // A non-network unit (a drive by-uuid .mount) must NOT parse as a network mount. + drive := renderMountUnit(MountSpec{Name: "usb", UUID: "0fc63daf-8483-4772-8e79-3d69d8477de4", Where: "/mnt/felhom-usb", FSType: "ext4"}) + if _, _, _, _, ok := parseNetworkMountUnit(drive); ok { + t.Errorf("a drive by-uuid unit must not parse as a network mount") + } +} + +func TestNetworkHealth(t *testing.T) { + cases := []struct { + reachable, mounted bool + want string + }{ + {true, true, NetHealthOK}, + {true, false, NetHealthIdle}, + {false, true, NetHealthUnreachable}, + {false, false, NetHealthUnreachable}, + } + for _, c := range cases { + if got := networkHealth(c.reachable, c.mounted); got != c.want { + t.Errorf("networkHealth(%v,%v)=%q want %q", c.reachable, c.mounted, got, c.want) + } + } +} + +func TestIsNetworkMounted(t *testing.T) { + for _, fs := range []string{"nfs", "nfs4", "cifs"} { + if !isNetworkMounted(fs) { + t.Errorf("%s should count as mounted", fs) + } + } + for _, fs := range []string{"autofs", "", "ext4", "tmpfs"} { + if isNetworkMounted(fs) { + t.Errorf("%s must NOT count as a real network mount (idle automount = autofs)", fs) + } + } +} + +// --- Scenario D: the drive machinery ignores a NAS mount (guard + companion red-proof) -------------- + +// TestNetMount_DriveMachineryGuard proves parseFelhomMountUnit (the host-reboot drive re-assert's +// classifier) REFUSES a network unit, so a NAS mount never enters the drive lifecycle. The companion +// red-proof: the SAME unit content WITHOUT the network marker but WITH a by-uuid What parses as a drive +// — i.e. it is the netUnitMarker guard (not luck) that keeps the NAS out of the drive machinery. +func TestNetMount_DriveMachineryGuard(t *testing.T) { + // A real network .mount unit: parseFelhomMountUnit must reject it. + netUnit := renderNetworkMountUnit(NetworkMountSpec{ + Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000, + }) + if _, ok := parseFelhomMountUnit(netUnit); ok { + t.Fatalf("a NAS network unit must NOT be classified as a drive by parseFelhomMountUnit:\n%s", netUnit) + } + + // Contrived worst case: a unit carrying the network marker AND a by-uuid What (the shape that WOULD + // otherwise parse as a drive). The guard must still refuse it. + contrived := "# " + netUnitMarker + "\n[Unit]\nDescription=Felhom storage mount x\n[Mount]\n" + + "What=" + byUUIDDir + "/0fc63daf-8483-4772-8e79-3d69d8477de4\nWhere=/mnt/felhom-drives/x\nType=nfs4\n" + if _, ok := parseFelhomMountUnit(contrived); ok { + t.Fatalf("the netUnitMarker guard must refuse a by-uuid-shaped network unit") + } + + // COMPANION RED-PROOF: identical content but with the drive marker instead of the network marker + // DOES parse as a drive — confirming the guard is the discriminator, not an accident of shape. + driveShaped := strings.Replace(contrived, "# "+netUnitMarker, "# "+felhomUnitMarker, 1) + if spec, ok := parseFelhomMountUnit(driveShaped); !ok || spec.UUID == "" { + t.Fatalf("control: a by-uuid unit with ONLY the drive marker should parse as a drive (ok=%v uuid=%q)", ok, spec.UUID) + } +} + +// --- SudoHostOps command sequence (Linux only — the unit filename embeds an escaped '-' = backslash) - + +func TestEnsureNetworkMount_Commands(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("systemd-escaped unit filename contains a backslash; staging is exercised on the Linux build server") + } + ctx := context.Background() + stage, unitDir := t.TempDir(), t.TempDir() + rr := &recordingRunner{} + ops := NewSudoHostOps(SudoHostOpsConfig{ + Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: stage, Logger: quietLogger(), + }) + spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000} + if err := ops.EnsureNetworkMount(ctx, spec); err != nil { + t.Fatalf("EnsureNetworkMount: %v", err) + } + // Expect: mkdir, install(.mount), install(.automount), daemon-reload, enable --now . + var sawMkdir, sawEnableAutomount, sawReload bool + installs := 0 + for _, c := range rr.calls { + joined := strings.Join(c, " ") + switch { + case strings.Contains(joined, "mkdir") && strings.Contains(joined, "/mnt/felhom-drives/media"): + sawMkdir = true + case strings.Contains(joined, "install"): + installs++ + case strings.Contains(joined, "daemon-reload"): + sawReload = true + case strings.Contains(joined, "enable") && strings.Contains(joined, "--now") && strings.Contains(joined, ".automount"): + sawEnableAutomount = true + } + } + if !sawMkdir || installs != 2 || !sawReload || !sawEnableAutomount { + t.Fatalf("unexpected command sequence (mkdir=%v installs=%d reload=%v enableAutomount=%v): %v", + sawMkdir, installs, sawReload, sawEnableAutomount, rr.calls) + } + // The .automount is enabled; the .mount is NOT (automount triggers it). + for _, c := range rr.calls { + joined := strings.Join(c, " ") + if strings.Contains(joined, "enable") && strings.Contains(joined, ".mount") && !strings.Contains(joined, ".automount") { + t.Errorf("the .mount unit must NOT be enabled (automount drives it): %v", c) + } + } + + // Staged units carry the right bodies. + autoName := "mnt-felhom\\x2ddrives-media.automount" + body, err := os.ReadFile(filepath.Join(stage, autoName)) + if err != nil { + t.Fatalf("staged automount not written: %v", err) + } + if !strings.Contains(string(body), "[Automount]") { + t.Errorf("staged automount missing [Automount]:\n%s", body) + } +} + +func TestEnsureNetworkMount_RejectsBadSpec(t *testing.T) { + ops := NewSudoHostOps(SudoHostOpsConfig{Runner: &recordingRunner{}, Bins: Binaries{}.withDefaults(), UnitDir: t.TempDir(), StageDir: t.TempDir(), Logger: quietLogger()}) + rr := ops.runner.(*recordingRunner) + if err := ops.EnsureNetworkMount(context.Background(), NetworkMountSpec{Name: "..", Protocol: ProtocolNFS, Server: "x", Export: "/y"}); err == nil { + t.Fatal("a bad spec must be refused") + } + if len(rr.calls) != 0 { + t.Fatalf("a refused spec must construct ZERO commands, got: %v", rr.calls) + } +} + +func TestRemoveNetworkMount_Commands(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server") + } + ctx := context.Background() + rr := &recordingRunner{} + ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: t.TempDir(), StageDir: t.TempDir(), Logger: quietLogger()}) + if err := ops.RemoveNetworkMount(ctx, "media"); err != nil { + t.Fatalf("RemoveNetworkMount: %v", err) + } + var stopAuto, disableAuto, rmCount, reload bool + rms := 0 + for _, c := range rr.calls { + j := strings.Join(c, " ") + switch { + case strings.Contains(j, "stop") && strings.Contains(j, ".automount"): + stopAuto = true + case strings.Contains(j, "disable") && strings.Contains(j, ".automount"): + disableAuto = true + case strings.Contains(j, "rm"): + rms++ + case strings.Contains(j, "daemon-reload"): + reload = true + } + } + rmCount = rms == 2 + if !stopAuto || !disableAuto || !rmCount || !reload { + t.Fatalf("unexpected remove sequence (stopAuto=%v disableAuto=%v rm=%d reload=%v): %v", stopAuto, disableAuto, rms, reload, rr.calls) + } +}