agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/storage/hostops.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -98,12 +99,12 @@ type MountSpec struct {
 // Binaries holds the absolute paths of the allow-listed binaries (overridable from config
 // so the sudoers entries and the agent agree on exact paths).
 type Binaries struct {
-	Systemctl string
-	Install   string
-	Smartctl  string
-	Lvs       string
-	Blkid     string // device signature probe (8C data-bearing detection)
-	Lsblk     string // partition/mount topology (8C)
+	Systemctl   string
+	Install     string
+	Smartctl    string
+	Lvs         string
+	Blkid       string // device signature probe (8C data-bearing detection)
+	Lsblk       string // partition/mount topology (8C)
 	MkfsExt4    string // 8C format executor (ext4) — now invoked by the guarded wrapper, not the agent directly
 	MkfsXfs     string // 8C format executor (xfs) — now invoked by the guarded wrapper, not the agent directly
 	MkfsGuarded string // Impl-1 Part B: the guarded-mkfs wrapper the agent execs (device+fstype)
@@ -155,18 +156,22 @@ func (b Binaries) withDefaults() Binaries {
 type SudoHostOps struct {
 	runner   proxmox.Runner
 	bins     Binaries
-	unitDir  string // where enabled units live (e.g. /etc/systemd/system)
-	stageDir string // agent-owned staging dir for unit files before install
+	unitDir  string     // where enabled units live (e.g. /etc/systemd/system)
+	stageDir string     // agent-owned staging dir for unit files before install
 	host     HostReader // root-free reads (mount table) for the Impl-1 Format claim guard
 	logger   *slog.Logger
+	// unitFailed reports whether a systemd unit is in the failed state (incl. start-limit-hit). An
+	// UNPRIVILEGED read (`systemctl is-failed`) — seam-injected so the reassert's F10 reset-failed path
+	// is unit-testable without a real systemd. Default set in NewSudoHostOps.
+	unitFailed func(ctx context.Context, unit string) bool
 }
 
 // SudoHostOpsConfig configures a SudoHostOps.
 type SudoHostOpsConfig struct {
 	Runner   proxmox.Runner
 	Bins     Binaries
-	UnitDir  string // default /etc/systemd/system
-	StageDir string // default <dataDir>/units; must be agent-writable
+	UnitDir  string     // default /etc/systemd/system
+	StageDir string     // default <dataDir>/units; must be agent-writable
 	Host     HostReader // default NewProcHostReader(); the Format claim guard's mount-table read
 	Logger   *slog.Logger
 }
@@ -190,15 +195,24 @@ func NewSudoHostOps(cfg SudoHostOpsConfig) *SudoHostOps {
 		host = NewProcHostReader()
 	}
 	return &SudoHostOps{
-		runner:   cfg.Runner,
-		bins:     cfg.Bins.withDefaults(),
-		unitDir:  unitDir,
-		stageDir: stageDir,
-		host:     host,
-		logger:   logger,
+		runner:     cfg.Runner,
+		bins:       cfg.Bins.withDefaults(),
+		unitDir:    unitDir,
+		stageDir:   stageDir,
+		host:       host,
+		logger:     logger,
+		unitFailed: systemctlIsFailed,
 	}
 }
 
+// systemctlIsFailed is the production unitFailed: `systemctl is-failed <unit>` prints "failed" for a
+// failed/start-limit-hit unit and exits nonzero otherwise. UNPRIVILEGED (unit state is world-readable)
+// — deliberately NOT routed through the sudo runner, so it needs no sudoers grant.
+func systemctlIsFailed(ctx context.Context, unit string) bool {
+	out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", "--", unit).Output()
+	return strings.TrimSpace(string(out)) == "failed"
+}
+
 // EnsureMount validates, renders, stages, installs and enables the .mount unit.
 func (h *SudoHostOps) EnsureMount(ctx context.Context, spec MountSpec) error {
 	// VALIDATE FIRST — refuse before constructing any command.

internal/storage/netmigrate_test.go

@@ -0,0 +1,170 @@
+package storage
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// F12 (CAMPAIGN-3, CRITICAL): NEITHER rendered network-storage unit may carry a network-online.target
+// ordering — that is exactly what closed the boot ordering cycle. The .mount keeps `_netdev` (the
+// correct, sufficient network ordering for the real mount). Companion red-proof: re-adding either
+// `After=`/`Wants=network-online.target` line to a template makes these assertions fail.
+func TestRenderNetworkUnits_NoNetworkOnlineOrdering(t *testing.T) {
+	for _, spec := range []NetworkMountSpec{
+		{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000},
+		{Name: "docs", Protocol: ProtocolSMB, Server: "10.0.0.6", Export: "share", CredsRef: "/var/lib/felhom-agent/smb-creds/docs.cred", MappedUID: 1000, MappedGID: 1000},
+	} {
+		mountUnit := renderNetworkMountUnit(spec)
+		autoUnit := renderNetworkAutomountUnit(spec)
+		if strings.Contains(mountUnit, "network-online.target") {
+			t.Errorf("[%s] .mount unit still orders network-online.target (F12 cycle):\n%s", spec.Name, mountUnit)
+		}
+		if strings.Contains(autoUnit, "network-online.target") {
+			t.Errorf("[%s] .automount unit still orders network-online.target (F12 cycle):\n%s", spec.Name, autoUnit)
+		}
+		if !strings.Contains(mountUnit, "_netdev") {
+			t.Errorf("[%s] .mount unit lost _netdev — the ONLY correct network ordering for the real mount:\n%s", spec.Name, mountUnit)
+		}
+	}
+}
+
+// specFromNetworkUnits must reconstruct a spec that re-renders EXACTLY the installed unit pair — the
+// idempotency contract of the drift reconcile. A round-trip that isn't byte-exact would make
+// MigrateNetworkUnits rewrite on every pass (never converging).
+func TestSpecFromNetworkUnits_RoundTrips(t *testing.T) {
+	for _, spec := range []NetworkMountSpec{
+		{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 60},
+		{Name: "docs", Protocol: ProtocolSMB, Server: "10.0.0.6", Export: "share", CredsRef: "/var/lib/felhom-agent/smb-creds/docs.cred", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 120},
+	} {
+		mountUnit := renderNetworkMountUnit(spec)
+		autoUnit := renderNetworkAutomountUnit(spec)
+		got, ok := specFromNetworkUnits(mountUnit, autoUnit)
+		if !ok {
+			t.Fatalf("[%s] specFromNetworkUnits failed to parse its own render", spec.Name)
+		}
+		if renderNetworkMountUnit(got) != mountUnit {
+			t.Errorf("[%s] .mount round-trip mismatch:\nWANT:\n%s\nGOT:\n%s", spec.Name, mountUnit, renderNetworkMountUnit(got))
+		}
+		if renderNetworkAutomountUnit(got) != autoUnit {
+			t.Errorf("[%s] .automount round-trip mismatch:\nWANT:\n%s\nGOT:\n%s", spec.Name, autoUnit, renderNetworkAutomountUnit(got))
+		}
+	}
+}
+
+// legacyNetworkUnitPair renders the pre-0.85 units WITH the F12 network-online ordering, the exact
+// drift the migration must repair.
+func legacyNetworkUnitPair(spec NetworkMountSpec) (mount, auto string) {
+	mount = renderNetworkMountUnit(spec)
+	mount = strings.Replace(mount, "\n[Mount]\n", "\nAfter=network-online.target\nWants=network-online.target\n[Mount]\n", 1)
+	auto = renderNetworkAutomountUnit(spec)
+	auto = strings.Replace(auto, "\n[Automount]\n", "\nAfter=network-online.target\nWants=network-online.target\n[Automount]\n", 1)
+	return mount, auto
+}
+
+// MigrateNetworkUnits rewrites a drifted (legacy, network-online-carrying) unit pair exactly ONCE,
+// batches a single daemon-reload, and is idempotent (a second pass rewrites nothing). A clean unit is
+// left untouched.
+func TestMigrateNetworkUnits_RewritesDriftedOnceIdempotent(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	stageDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 60}
+	mountName, err := UnitNameForMount(spec.Where())
+	if err != nil {
+		t.Fatalf("unit name: %v", err)
+	}
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	legacyMount, legacyAuto := legacyNetworkUnitPair(spec)
+	writeFile(t, filepath.Join(unitDir, mountName), legacyMount)
+	writeFile(t, filepath.Join(unitDir, autoName), legacyAuto)
+
+	rr := &recordingRunner{}
+	// installUnit stages then `install`s (recorded, not executed); rewrite the unit dir copy ourselves
+	// so the on-disk content reflects the migration for the idempotency re-read.
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: stageDir, Host: &fakeHostReader{}, Logger: quietLogger()})
+
+	migrated := ops.MigrateNetworkUnits(context.Background())
+	if migrated != 1 {
+		t.Fatalf("first pass must migrate exactly 1 unit, got %d", migrated)
+	}
+	// The recorded calls must include exactly one daemon-reload (batched), plus the two installs.
+	reloads, installs := 0, 0
+	for _, c := range rr.calls {
+		joined := strings.Join(c, " ")
+		if strings.Contains(joined, "daemon-reload") {
+			reloads++
+		}
+		if strings.Contains(joined, "install") {
+			installs++
+		}
+	}
+	if reloads != 1 {
+		t.Errorf("migration must batch exactly ONE daemon-reload, got %d (calls: %v)", reloads, rr.calls)
+	}
+	if installs != 2 {
+		t.Errorf("migration must rewrite both units (2 installs), got %d", installs)
+	}
+	// The staged content the installer would have placed must be the CLEAN template. Simulate the
+	// install landing (installUnit staged to stageDir/<unit>), then re-read for idempotency.
+	applyStaged(t, stageDir, unitDir, mountName)
+	applyStaged(t, stageDir, unitDir, autoName)
+	if got := readFile(t, filepath.Join(unitDir, mountName)); strings.Contains(got, "network-online.target") {
+		t.Errorf("migrated .mount still carries network-online.target:\n%s", got)
+	}
+
+	rr.calls = nil
+	if migrated := ops.MigrateNetworkUnits(context.Background()); migrated != 0 {
+		t.Fatalf("second pass over already-current units must migrate 0, got %d", migrated)
+	}
+	if len(rr.calls) != 0 {
+		t.Errorf("idempotent second pass must construct ZERO commands, got: %v", rr.calls)
+	}
+}
+
+// A non-marker unit file in the unit dir is never touched by the migration.
+func TestMigrateNetworkUnits_IgnoresForeignUnits(t *testing.T) {
+	unitDir := t.TempDir()
+	writeFile(t, filepath.Join(unitDir, "some-service.mount"), "[Unit]\nDescription=not ours\n[Mount]\nWhere=/x\n")
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
+	if migrated := ops.MigrateNetworkUnits(context.Background()); migrated != 0 {
+		t.Fatalf("a foreign unit must not be migrated, got %d", migrated)
+	}
+	if len(rr.calls) != 0 {
+		t.Errorf("a foreign unit must construct zero commands, got: %v", rr.calls)
+	}
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(b)
+}
+
+// applyStaged mirrors what the (recorded, not executed) `install` would do: copy the agent-staged unit
+// into the unit dir, so the idempotency re-read sees the migrated content.
+func applyStaged(t *testing.T, stageDir, unitDir, unitName string) {
+	t.Helper()
+	src := filepath.Join(stageDir, unitName)
+	b, err := os.ReadFile(src)
+	if err != nil {
+		return // installUnit stages before the recorded install; if absent, nothing to mirror
+	}
+	writeFile(t, filepath.Join(unitDir, unitName), string(b))
+}

internal/storage/netmount.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"os"
@@ -246,8 +248,12 @@ func renderNetworkMountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): NO network-online.target ordering here. `_netdev` in Options is the
+	// correct + sufficient network ordering for the REAL mount — systemd classes a _netdev mount under
+	// remote-fs.target and orders it after the network without a hand-written After/Wants. A literal
+	// `After=network-online.target` on this unit (which the .automount pulls in via local-fs) closed the
+	// boot ordering cycle networking→local-fs→automount→network-online→networking; systemd broke it by
+	// DELETING an arbitrary job (one boot lost networking entirely, the next lost the automount).
 	b.WriteString("\n[Mount]\n")
 	fmt.Fprintf(&b, "What=%s\n", s.mountSource())
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
@@ -263,8 +269,12 @@ func renderNetworkAutomountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage automount %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): the automount unit gets NO network relation of ANY kind. A trigger
+	// needs no network — it just watches the mountpoint and fires the .mount on first access (the
+	// .mount's `_netdev` then orders the real mount after the network). An automount is implicitly
+	// ordered Before=local-fs.target; adding After/Wants=network-online.target here created the boot
+	// ordering cycle that cost the host its network on one boot and its NAS on the next. Keep this unit
+	// orderable before local-fs WITHOUT dragging the network into that transaction.
 	b.WriteString("\n[Automount]\n")
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
 	fmt.Fprintf(&b, "TimeoutIdleSec=%d\n", s.idleTimeout())
@@ -347,6 +357,10 @@ func (h *SudoHostOps) EnsureNetworkMount(ctx context.Context, spec NetworkMountS
 	if err := ValidateNetworkMountSpec(spec); err != nil {
 		return err
 	}
+	// Template-drift reconcile: bring any already-installed units up to the current template before we
+	// touch the unit dir (F12 — a pre-0.85 unit still carrying the network-online ordering gets rewritten
+	// here even if the daemon-startup migration hasn't run in this process). Best-effort; never blocks add.
+	h.MigrateNetworkUnits(ctx)
 	// Defense in depth: never realise a network mount outside the user-data namespace.
 	if NetworkMountRole(spec.Where()) != RoleUserData {
 		return fmt.Errorf("netmount: refusing to mount outside the user-data namespace: %s", spec.Where())
@@ -427,6 +441,10 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 		h.logger.Debug("netmount: remove step", "verb", step[0], "unit", step[len(step)-1],
 			"ok", err == nil) // a "not loaded" failure here is expected + tolerated
 	}
+	// F2 (CAMPAIGN-3): clear any failed/start-limit runtime state on the pair BEFORE the files go, or
+	// systemd keeps them as `not-found failed` residue after daemon-reload. reset-failed while the units
+	// are still loaded; tolerate the not-failed case (nothing to reset).
+	h.resetNetworkUnitsIfFailed(ctx, automountUnit, mountUnit)
 
 	destAuto := filepath.Join(h.unitDir, automountUnit)
 	destMount := filepath.Join(h.unitDir, mountUnit)
@@ -441,10 +459,184 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 	if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
 		return fmt.Errorf("netmount: daemon-reload: %w", err)
 	}
+	// F1 (CAMPAIGN-3): remove the now-empty mountpoint directory (the campaign accumulated 10 stub-shaped
+	// leftovers). rmdir ONLY — a non-empty dir (unexpected data present) is left in place with a WARN, the
+	// fail-safe; `rm -rf` is forbidden here. The host removal propagates into running guests through the
+	// shared bind; a fresh guest re-binds cleanly on next start.
+	h.rmdirMountpoint(ctx, where)
 	h.logger.Info("netmount: removed network mount", "name", name, "where", where)
 	return nil
 }
 
+// resetNetworkUnitsIfFailed reset-failed's any of the given units that is in the failed state (F2 —
+// leave no `not-found failed`/start-limit residue behind a remove or a rolled-back add). Unprivileged
+// is-failed read + the FELHOM_NETMOUNT reset-failed grant; every step tolerated.
+func (h *SudoHostOps) resetNetworkUnitsIfFailed(ctx context.Context, units ...string) {
+	for _, unit := range units {
+		if h.unitFailed == nil || !h.unitFailed(ctx, unit) {
+			continue
+		}
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			h.logger.Warn("netmount: reset-failed tolerated failure", "unit", unit, "err", err)
+		}
+	}
+}
+
+// rmdirMountpoint removes an empty network mountpoint dir under NetworkMountRoot. rmdir refuses a
+// non-empty dir (the fail-safe): unexpected data is preserved and flagged, never rm -rf'd. Best-effort.
+func (h *SudoHostOps) rmdirMountpoint(ctx context.Context, where string) {
+	if !strings.HasPrefix(where, NetworkMountRoot+"/") {
+		return // defense in depth: only ever under the bind root
+	}
+	if err := h.run(ctx, "/usr/bin/rmdir", where); err != nil {
+		// rmdir fails on a non-empty dir — leave it (fail-safe) and flag it for the operator.
+		h.logger.Warn("netmount: mountpoint dir not removed (non-empty or busy — left in place, fail-safe)",
+			"where", where, "err", err)
+	}
+}
+
+// MigrateNetworkUnits reconciles every marker-owned network-storage unit file on disk against a fresh
+// render of its own reconstructed spec — a general template-drift reconcile (the git-sync pattern:
+// content-hash compare, rewrite on drift, batched daemon-reload). It exists because a template change
+// must reach ALREADY-INSTALLED units, not only future adds: the F12 fix (CAMPAIGN-3) removed the
+// network-online ordering that turned every host boot with an enrolled share into a coin flip, and the
+// units installed before 0.85 still carry the ordering cycle until they are rewritten. Runs at agent
+// startup (before the reassert sweep) and at the head of EnsureNetworkMount. Idempotent: a unit already
+// byte-identical to its fresh render is left untouched (second pass rewrites nothing). Best-effort per
+// unit; one INFO line per migrated unit. Returns the count migrated.
+func (h *SudoHostOps) MigrateNetworkUnits(ctx context.Context) int {
+	entries, err := os.ReadDir(h.unitDir)
+	if err != nil {
+		h.logger.Warn("netmigrate: reading unit dir failed", "err", err)
+		return 0
+	}
+	migrated := 0
+	changed := false
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".mount") {
+			continue // the .mount carries What/Type; the paired .automount mirrors Where
+		}
+		mountPath := filepath.Join(h.unitDir, e.Name())
+		mountData, rerr := os.ReadFile(mountPath)
+		if rerr != nil || !strings.Contains(string(mountData), netUnitMarker) {
+			continue // unreadable or not one of ours
+		}
+		automountName := strings.TrimSuffix(e.Name(), ".mount") + ".automount"
+		automountData, aerr := os.ReadFile(filepath.Join(h.unitDir, automountName))
+		if aerr != nil {
+			continue // a .mount with no paired .automount is malformed — not ours to guess
+		}
+		spec, ok := specFromNetworkUnits(string(mountData), string(automountData))
+		if !ok {
+			continue
+		}
+		freshMount := renderNetworkMountUnit(spec)
+		freshAuto := renderNetworkAutomountUnit(spec)
+		if contentHash(string(mountData)) == contentHash(freshMount) &&
+			contentHash(string(automountData)) == contentHash(freshAuto) {
+			continue // already current — the idempotent no-op
+		}
+		if err := h.installUnit(ctx, e.Name(), freshMount); err != nil {
+			h.logger.Warn("netmigrate: rewriting mount unit failed", "unit", e.Name(), "err", err)
+			continue
+		}
+		if err := h.installUnit(ctx, automountName, freshAuto); err != nil {
+			h.logger.Warn("netmigrate: rewriting automount unit failed", "unit", automountName, "err", err)
+			continue
+		}
+		changed = true
+		migrated++
+		h.logger.Info("netmigrate: migrated network-storage unit to the current template (F12: dropped the boot ordering cycle)",
+			"name", spec.Name, "where", spec.Where())
+	}
+	if changed {
+		if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
+			h.logger.Warn("netmigrate: daemon-reload after migration failed", "err", err)
+		}
+	}
+	return migrated
+}
+
+// contentHash is the SHA-256 hex of a unit file's content — the drift comparator (git-sync pattern).
+func contentHash(s string) string {
+	sum := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(sum[:])
+}
+
+// specFromNetworkUnits reconstructs the NetworkMountSpec that renders EXACTLY the given installed unit
+// pair — the fresh-render input for the drift reconcile. Round-trips by construction: every field the
+// render templates read is recovered (proto/server/export/where from the .mount; the SMB uid/gid+creds
+// from its Options; the idle window from the .automount). A NFS spec's mapped uid never appears in a
+// rendered unit, so it is irrelevant to the render and left at the container default. ok=false for a
+// non-marker or unparseable pair.
+func specFromNetworkUnits(mountContent, automountContent string) (NetworkMountSpec, bool) {
+	proto, server, export, where, ok := parseNetworkMountUnit(mountContent)
+	if !ok {
+		return NetworkMountSpec{}, false
+	}
+	spec := NetworkMountSpec{
+		Name:      strings.TrimPrefix(where, NetworkMountRoot+"/"),
+		Protocol:  NetworkProtocol(proto),
+		Server:    server,
+		Export:    export,
+		MappedUID: 1000, // container default; unused by the NFS render, overwritten below for SMB
+		MappedGID: 1000,
+	}
+	if spec.Protocol == ProtocolSMB {
+		opts := unitLineValue(mountContent, "Options=")
+		if uid, ok := csvIntField(opts, "uid="); ok {
+			spec.MappedUID = uid - lxcUIDOffset
+		}
+		if gid, ok := csvIntField(opts, "gid="); ok {
+			spec.MappedGID = gid - lxcUIDOffset
+		}
+		spec.CredsRef = csvField(opts, "credentials=")
+	}
+	if idle, ok := csvIntField(unitLineValue(automountContent, "TimeoutIdleSec="), ""); ok && idle > 0 {
+		spec.IdleTimeoutSec = idle
+	}
+	return spec, true
+}
+
+// unitLineValue returns the value after the first line beginning with prefix (e.g. "Options="), trimmed.
+func unitLineValue(content, prefix string) string {
+	for _, line := range strings.Split(content, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, prefix) {
+			return strings.TrimPrefix(line, prefix)
+		}
+	}
+	return ""
+}
+
+// csvField finds the comma-separated token with the given key prefix (e.g. "credentials=") and returns
+// its value. "" if absent.
+func csvField(csv, key string) string {
+	for _, tok := range strings.Split(csv, ",") {
+		if strings.HasPrefix(tok, key) {
+			return strings.TrimPrefix(tok, key)
+		}
+	}
+	return ""
+}
+
+// csvIntField parses the int value of a comma-separated key (e.g. "uid=") — or, when key is "", parses
+// the whole string as an int (for a bare value like TimeoutIdleSec's already-extracted number).
+func csvIntField(csv, key string) (int, bool) {
+	val := csv
+	if key != "" {
+		val = csvField(csv, key)
+	}
+	if val == "" {
+		return 0, false
+	}
+	n, err := strconv.Atoi(strings.TrimSpace(val))
+	if err != nil {
+		return 0, false
+	}
+	return n, true
+}
+
 // ListNetworkMounts enumerates the installed network-storage units and reports per-share liveness. It
 // reads the (world-readable) unit dir + /proc/mounts and TCP-probes each NAS endpoint with a short
 // timeout — it NEVER stat()s the (possibly EIO/D-state) mountpoint, so a black-holed NAS cannot wedge

internal/storage/netreassert.go

@@ -6,29 +6,43 @@ import (
 	"strings"
 )
 
-// Network-mount guest-reboot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1).
+// Network-mount guest-reboot / boot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1, hardened by
+// CAMPAIGN-3 F9/F10/F11).
 //
-// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or
-// an actively-mounted nfs4) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
+// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or an
+// actively-mounted nfs4/cifs) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
 // share silently degrades to a local stub directory inside the guest. The heal is host-side and
-// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates
-// live into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
+// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates live
+// into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
 //
-// The action uses the sudoers-granted verbs only (`systemctl stop -- *.automount` +
-// `systemctl enable --now -- *.automount`; there is NO restart grant). Idempotent: re-arming an
-// already-armed trigger just recreates it — same end state, and the fresh mount event is harmless.
-// An ACTIVE real mount is never touched (stopping the automount of a live mount would churn it).
+// CAMPAIGN-3 hardening:
+//   - F11 (read the right unit): the decision is driven ONLY by the host `/proc/mounts` fstype AT the
+//     mountpoint — an ACTIVE real mount (nfs4/cifs) is inherited and left alone; anything else is
+//     re-armed. The `.automount` unit's own state is NEVER consulted (an armed trigger always reports
+//     "active", which is exactly why a state-of-the-automount check mis-skips idle triggers).
+//   - F10 (re-arm for real): a `.mount`/`.automount` left in `failed`/start-limit-hit state (the
+//     campaign's unexport→idle-timeout→access×5 sequence) is `reset-failed` FIRST — without it the
+//     `enable --now` below is refused by the start limit and the share stays dead across every boot.
+//   - F9 (say what you did): the pass enumerates by the marker-owned unit files on disk (not by
+//     enablement or runtime state) and logs an INFO verdict line for EVERY share — an empty-looking
+//     sweep over N shares is structurally impossible.
+//
+// The action uses the sudoers-granted verbs only (`systemctl reset-failed -- *`, `systemctl stop -- *`,
+// `systemctl enable --now -- *`). Idempotent: re-arming an already-armed trigger recreates it — same end
+// state, and the fresh mount event is harmless. An ACTIVE real mount is never touched.
 
-// Reassert actions (the §8 decision table, encoded).
+// Reassert actions (the §8 decision table, encoded — CAMPAIGN-3 F11).
 const (
 	// NetReassertRearmed: the trigger was re-created (stop + enable --now) — the propagation heal.
 	NetReassertRearmed = "rearmed"
-	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh
-	// namespaces, nothing to do (verify only).
+	// NetReassertResetRearmed: the unit was failed/start-limited, reset-failed, THEN re-armed (F10).
+	NetReassertResetRearmed = "reset-failed+rearmed"
+	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh namespaces,
+	// nothing to do (verify only).
 	NetReassertSkipActive = "skip-active"
-	// NetReassertSkipNone: neither a real mount nor an armed trigger at the path — a removed or
-	// orphan state owned by the add/remove flows, not this reconcile.
-	NetReassertSkipNone = "skip-none"
+	// NetReassertSkipForeign: a non-network, non-autofs filesystem occupies the path (ext4/tmpfs/…) —
+	// not a state this reconcile owns, and re-arming over it would fail (mountpoint busy).
+	NetReassertSkipForeign = "skip-foreign"
 )
 
 // NetReassertResult is one share's outcome in a reassert pass.
@@ -39,24 +53,36 @@ type NetReassertResult struct {
 	Err    error  // set when the rearm action failed (skip rows never error)
 }
 
-// netReassertAction is the pure §8 decision: the /proc/mounts fstype at the share's mountpoint
-// ("" = nothing mounted there) → the action to take.
-func netReassertAction(fstype string) string {
+// Remediates reports whether an action expects the share to become visible in running guests (drives
+// the caller's guest-visibility verify). Skip-active also expects visibility (an inherited live mount);
+// only foreign-fs and errored rows expect nothing.
+func (r NetReassertResult) Remediates() bool {
+	return r.Err == nil && r.Action != NetReassertSkipForeign
+}
+
+// netReassertActive reports whether the fstype at a share's mountpoint is a live network mount — the
+// ONLY input to the skip-active decision (F11: never the automount unit's state). "" (a failed/disarmed
+// automount leaves NO /proc/mounts entry) and "autofs" (an armed-but-idle trigger) are BOTH not-active
+// and therefore re-arm targets; a foreign local fs is left alone.
+func netReassertClassify(fstype string) string {
 	switch {
 	case isNetworkMounted(fstype):
 		return NetReassertSkipActive
-	case fstype == "autofs":
+	case fstype == "" || fstype == "autofs":
+		// Not actively mounted, but a marker unit exists for this path: idle-armed, disarmed, OR
+		// failed/start-limited — all of them must be re-armed so a fresh trigger event propagates.
 		return NetReassertRearmed
 	default:
-		return NetReassertSkipNone
+		return NetReassertSkipForeign // ext4/tmpfs/… — foreign, not ours to churn
 	}
 }
 
 // ReassertNetworkAutomounts runs the reassert pass over every configured network mount: for each
-// installed pair, decide per netReassertAction and re-arm idle triggers. Returns one result per
-// share so callers (agent startup / guest-hook post-start) can verify guest visibility. Errors on
-// one share never stop the pass. Callers MUST NOT invoke this from periodic health paths — an idle
-// trigger is healthy, and the pass is only needed after a guest (re)start or at agent startup.
+// installed pair, decide per netReassertClassify and re-arm every not-active trigger (reset-failed first
+// if the unit is stuck). Returns one result per share so callers (agent startup / guest-hook post-start)
+// can verify guest visibility. Errors on one share never stop the pass. Callers MUST NOT invoke this
+// from periodic health paths — an idle trigger is healthy, and the pass is only needed after a guest
+// (re)start or at agent startup.
 func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReassertResult {
 	entries, err := h.networkUnitEntries()
 	if err != nil {
@@ -75,21 +101,27 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	}
 	var out []NetReassertResult
 	for _, e := range entries {
-		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertAction(fstypes[e.where])}
+		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertClassify(fstypes[e.where])}
 		switch res.Action {
 		case NetReassertSkipActive:
-			h.logger.Debug("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
-				"name", e.name, "where", e.where)
-		case NetReassertSkipNone:
-			h.logger.Debug("netreassert: no mount and no armed trigger — skip (removed/orphan state owned elsewhere)",
-				"name", e.name, "where", e.where)
+			h.logger.Info("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
+		case NetReassertSkipForeign:
+			h.logger.Info("netreassert: foreign filesystem at mountpoint — skip (not a network state we own)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
 		case NetReassertRearmed:
+			// F10: clear a failed/start-limit lockout FIRST or the enable --now is refused; the verdict
+			// records whether a reset was actually needed.
+			if h.resetNetworkAutomountIfFailed(ctx, e.where) {
+				res.Action = NetReassertResetRearmed
+			}
 			if err := h.rearmNetworkAutomount(ctx, e.where); err != nil {
 				res.Err = err
-				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where, "err", err)
+				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where,
+					"verdict", "error", "err", err)
 			} else {
 				h.logger.Info("netreassert: automount trigger re-armed (fresh mount event propagates into running guests)",
-					"name", e.name, "where", e.where)
+					"name", e.name, "where", e.where, "verdict", res.Action)
 			}
 		}
 		out = append(out, res)
@@ -97,6 +129,33 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	return out
 }
 
+// resetNetworkAutomountIfFailed clears a failed/start-limit-hit lockout on the share's unit pair so the
+// subsequent `enable --now` is not refused (F10 — the campaign's start-limited automount that no
+// platform path re-armed). Returns true when either unit was in the failed state (so the caller can
+// report the reset-failed+rearmed verdict). The failed-state read is unprivileged (`systemctl
+// is-failed`, seam-injected); the reset-failed is the new sudoers verb.
+func (h *SudoHostOps) resetNetworkAutomountIfFailed(ctx context.Context, where string) bool {
+	mountUnit, err := UnitNameForMount(where)
+	if err != nil {
+		return false
+	}
+	automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount"
+	reset := false
+	for _, unit := range []string{automountUnit, mountUnit} {
+		if !h.unitFailed(ctx, unit) {
+			continue
+		}
+		reset = true
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			// Tolerated: a reset-failed that itself fails still lets the enable --now try; log it.
+			h.logger.Warn("netreassert: reset-failed tolerated failure", "unit", unit, "err", err)
+		} else {
+			h.logger.Warn("netreassert: cleared failed/start-limit lockout before re-arm (F10)", "unit", unit)
+		}
+	}
+	return reset
+}
+
 // rearmNetworkAutomount stops then re-enables+starts the .automount for a mountpoint. The stop is
 // tolerated failing (unit not loaded); the enable --now is the action that must succeed. Both verbs
 // are the existing FELHOM_NETMOUNT sudoers grants.

internal/storage/netreassert_test.go

@@ -9,23 +9,23 @@ import (
 	"testing"
 )
 
-// The §8 decision table, encoded exactly (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1).
-func TestNetReassertAction_Table(t *testing.T) {
+// The §8 decision table, encoded exactly (CAMPAIGN-3 F11: fstype-driven, automount state IGNORED).
+func TestNetReassertClassify_Table(t *testing.T) {
 	cases := []struct {
 		fstype string
 		want   string
 	}{
-		{"nfs4", NetReassertSkipActive},  // real mount — inherited by fresh namespaces
-		{"nfs", NetReassertSkipActive},   // real mount
-		{"cifs", NetReassertSkipActive},  // real mount
-		{"autofs", NetReassertRearmed},   // idle trigger — NOT inherited, re-arm to propagate
-		{"", NetReassertSkipNone},        // nothing at the path — removed/orphan, owned elsewhere
-		{"ext4", NetReassertSkipNone},    // a local fs at the path is not a network state we own
-		{"tmpfs", NetReassertSkipNone},   //
+		{"nfs4", NetReassertSkipActive},   // real mount — inherited by fresh namespaces
+		{"nfs", NetReassertSkipActive},    // real mount
+		{"cifs", NetReassertSkipActive},   // real mount
+		{"autofs", NetReassertRearmed},    // idle trigger — NOT inherited, re-arm to propagate
+		{"", NetReassertRearmed},          // F10: a failed/disarmed automount leaves NO mount entry — re-arm
+		{"ext4", NetReassertSkipForeign},  // a foreign local fs at the path is not ours to churn
+		{"tmpfs", NetReassertSkipForeign}, //
 	}
 	for _, c := range cases {
-		if got := netReassertAction(c.fstype); got != c.want {
-			t.Errorf("netReassertAction(%q) = %q, want %q", c.fstype, got, c.want)
+		if got := netReassertClassify(c.fstype); got != c.want {
+			t.Errorf("netReassertClassify(%q) = %q, want %q", c.fstype, got, c.want)
 		}
 	}
 }
@@ -45,18 +45,26 @@ func installNetUnitFile(t *testing.T, unitDir string, spec NetworkMountSpec) (wh
 	return where, strings.TrimSuffix(mountUnit, ".mount") + ".automount"
 }
 
-func netReassertOps(t *testing.T, unitDir string, mounts []Mount) (*SudoHostOps, *recordingRunner) {
+// netReassertOps builds ops with a hermetic unitFailed seam (default: nothing failed — no shelling out
+// to a real systemctl is-failed). failedUnits, if set, marks specific unit names as failed.
+func netReassertOps(t *testing.T, unitDir string, mounts []Mount, failedUnits ...string) (*SudoHostOps, *recordingRunner) {
 	t.Helper()
 	rr := &recordingRunner{}
 	ops := NewSudoHostOps(SudoHostOpsConfig{
 		Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(),
 		Host: &fakeHostReader{mounts: mounts}, Logger: quietLogger(),
 	})
+	failed := map[string]bool{}
+	for _, u := range failedUnits {
+		failed[u] = true
+	}
+	ops.unitFailed = func(_ context.Context, unit string) bool { return failed[unit] }
 	return ops, rr
 }
 
 // An idle trigger (autofs at the mountpoint) must be re-armed with EXACTLY the granted verbs:
-// `systemctl stop -- <unit>.automount` then `systemctl enable --now -- <unit>.automount`.
+// `systemctl stop -- <unit>.automount` then `systemctl enable --now -- <unit>.automount`. No
+// reset-failed when nothing is failed.
 func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -70,11 +78,8 @@ func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	if len(results) != 1 || results[0].Action != NetReassertRearmed || results[0].Err != nil {
 		t.Fatalf("want one rearmed result, got %+v", results)
 	}
-	if results[0].Where != where || results[0].Name != "media" {
-		t.Fatalf("result identity wrong: %+v", results[0])
-	}
 	if len(rr.calls) != 2 {
-		t.Fatalf("want exactly stop + enable --now, got %d calls: %v", len(rr.calls), rr.calls)
+		t.Fatalf("want exactly stop + enable --now (no reset-failed when clean), got %d calls: %v", len(rr.calls), rr.calls)
 	}
 	stop, enable := strings.Join(rr.calls[0], " "), strings.Join(rr.calls[1], " ")
 	if !strings.Contains(stop, "systemctl stop -- "+autoUnit) {
@@ -85,8 +90,44 @@ func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	}
 }
 
+// F10: a failed/start-limited automount (the campaign's unexport→idle-timeout→access×5 sequence leaves
+// NO mount entry, fstype "") must be reset-failed FIRST, then re-armed — verdict reset-failed+rearmed.
+// Companion to the campaign's reboots #2–#4: the pre-0.85 code returned skip-none here and left it dead.
+func TestReassertNetworkAutomounts_ResetsFailedThenRearms(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where, autoUnit := installNetUnitFile(t, unitDir, spec)
+	// Nothing mounted (fstype "" — a start-limited automount), and the automount unit is in failed state.
+	ops, rr := netReassertOps(t, unitDir, nil, autoUnit)
+
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != 1 || results[0].Action != NetReassertResetRearmed || results[0].Err != nil {
+		t.Fatalf("want one reset-failed+rearmed result, got %+v", results)
+	}
+	// reset-failed <automount>, then stop + enable --now.
+	var sawReset, sawEnable bool
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		if strings.Contains(j, "reset-failed -- "+autoUnit) {
+			sawReset = true
+		}
+		if strings.Contains(j, "enable --now -- "+autoUnit) {
+			sawEnable = true
+		}
+	}
+	if !sawReset {
+		t.Errorf("a failed unit must be reset-failed before re-arm (F10); calls: %v", rr.calls)
+	}
+	if !sawEnable {
+		t.Errorf("the trigger must still be re-armed after reset-failed; calls: %v", rr.calls)
+	}
+}
+
 // An ACTIVE real mount must not be touched — stopping the automount of a live mount would churn it.
-// (Red-proof companion: a naive always-rearm implementation fails this with 2 recorded calls.)
+// (Red-proof companion: a naive always-rearm implementation fails this with recorded calls.)
 func TestReassertNetworkAutomounts_ActiveMountUntouched(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -105,27 +146,81 @@ func TestReassertNetworkAutomounts_ActiveMountUntouched(t *testing.T) {
 	}
 }
 
-// Neither a mount nor an armed trigger → skip (removed/orphan state, owned by add/remove flows).
-func TestReassertNetworkAutomounts_NoTriggerSkips(t *testing.T) {
+// The automount unit's own state is IGNORED (F11 red-proof): even though a mutant that consulted
+// `systemctl is-active <automount>` would see an armed trigger as "active" and skip it, the fstype at
+// the path is autofs (idle) so the correct code RE-ARMS. Encoded as: an idle trigger re-arms regardless
+// of failed/armed unit state — the decision is fstype only.
+func TestReassertNetworkAutomounts_IgnoresAutomountUnitState(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
 	}
 	unitDir := t.TempDir()
 	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
-	installNetUnitFile(t, unitDir, spec)
-	ops, rr := netReassertOps(t, unitDir, nil) // nothing at the mountpoint
-
+	where, _ := installNetUnitFile(t, unitDir, spec)
+	// fstype autofs (idle-armed): the correct decision is re-arm, NOT skip — a state-of-the-automount
+	// check would mis-skip an armed trigger (which always reports "active").
+	ops, _ := netReassertOps(t, unitDir, []Mount{{MountPoint: where, FSType: "autofs"}})
 	results := ops.ReassertNetworkAutomounts(context.Background())
-	if len(results) != 1 || results[0].Action != NetReassertSkipNone {
-		t.Fatalf("want one skip-none result, got %+v", results)
-	}
-	if len(rr.calls) != 0 {
-		t.Fatalf("a unit with no trigger must not be acted on, got: %v", rr.calls)
+	if len(results) != 1 || (results[0].Action != NetReassertRearmed && results[0].Action != NetReassertResetRearmed) {
+		t.Fatalf("an idle (autofs) trigger must re-arm regardless of automount unit state, got %+v", results)
 	}
 }
 
-// Idempotency: two consecutive passes over an idle trigger both succeed with the same action and no
-// error (re-arming a fresh trigger is harmless — same end state).
+// A foreign filesystem at the path (ext4/tmpfs) is skipped — re-arming over it would fail (busy).
+func TestReassertNetworkAutomounts_ForeignFSSkipped(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where, _ := installNetUnitFile(t, unitDir, spec)
+	ops, rr := netReassertOps(t, unitDir, []Mount{{MountPoint: where, FSType: "ext4"}})
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != 1 || results[0].Action != NetReassertSkipForeign {
+		t.Fatalf("want one skip-foreign result, got %+v", results)
+	}
+	if len(rr.calls) != 0 {
+		t.Fatalf("a foreign fs at the path must trigger zero systemctl calls, got: %v", rr.calls)
+	}
+}
+
+// F9: the pass returns exactly one verdict per installed unit — including a failed one. An
+// empty-looking sweep over N shares is structurally impossible (the campaign's silent zero-line sweep).
+func TestReassertNetworkAutomounts_VerdictPerUnit(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	specs := []NetworkMountSpec{
+		{Name: "alpha", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/a", MappedUID: 1000, MappedGID: 1000},
+		{Name: "beta", Protocol: ProtocolNFS, Server: "10.0.0.6", Export: "/srv/b", MappedUID: 1000, MappedGID: 1000},
+		{Name: "gamma", Protocol: ProtocolNFS, Server: "10.0.0.7", Export: "/srv/c", MappedUID: 1000, MappedGID: 1000},
+	}
+	for _, s := range specs {
+		installNetUnitFile(t, unitDir, s)
+	}
+	// alpha active (skip-active), beta idle (rearm), gamma failed-and-unmounted (reset-failed+rearm).
+	gammaAuto := func() string {
+		u, _ := UnitNameForMount(specs[2].Where())
+		return strings.TrimSuffix(u, ".mount") + ".automount"
+	}()
+	ops, _ := netReassertOps(t, unitDir, []Mount{
+		{MountPoint: specs[0].Where(), FSType: "nfs4"},
+		{MountPoint: specs[1].Where(), FSType: "autofs"},
+	}, gammaAuto)
+
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != len(specs) {
+		t.Fatalf("verdict count must equal unit count (%d), got %d: %+v", len(specs), len(results), results)
+	}
+	for _, r := range results {
+		if r.Action == "" {
+			t.Errorf("every share must carry a verdict (F9), got empty for %s", r.Name)
+		}
+	}
+}
+
+// Idempotency: two consecutive passes over an idle trigger both re-arm with the same action, no error.
 func TestReassertNetworkAutomounts_Idempotent(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -140,9 +235,6 @@ func TestReassertNetworkAutomounts_Idempotent(t *testing.T) {
 	if first[0].Action != NetReassertRearmed || second[0].Action != NetReassertRearmed {
 		t.Fatalf("both passes must re-arm: first=%+v second=%+v", first, second)
 	}
-	if first[0].Err != nil || second[0].Err != nil {
-		t.Fatalf("idempotent passes must not error: first=%v second=%v", first[0].Err, second[0].Err)
-	}
 	if len(rr.calls) != 4 {
 		t.Fatalf("two passes = 2×(stop+enable), got %d: %v", len(rr.calls), rr.calls)
 	}

internal/storage/netresidue_test.go

@@ -0,0 +1,96 @@
+package storage
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// F2/F1 (CAMPAIGN-3): a remove (and, via the same path, a rolled-back add) must leave ZERO residue —
+// no failed-state units, no leftover mountpoint dir. RemoveNetworkMount must reset-failed the stuck
+// unit BEFORE removing the files (or systemd keeps it as not-found/failed) and rmdir the mountpoint.
+func TestRemoveNetworkMount_ZeroResidue(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	stageDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where := spec.Where()
+	mountName, err := UnitNameForMount(where)
+	if err != nil {
+		t.Fatalf("unit name: %v", err)
+	}
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	// Both unit files present on disk (rm is recorded, so they stay for the assertion).
+	if err := os.WriteFile(filepath.Join(unitDir, mountName), []byte(renderNetworkMountUnit(spec)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(unitDir, autoName), []byte(renderNetworkAutomountUnit(spec)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: stageDir, Host: &fakeHostReader{}, Logger: quietLogger()})
+	// The automount is in the failed state (start-limit residue) — must be reset-failed.
+	ops.unitFailed = func(_ context.Context, unit string) bool { return unit == autoName }
+
+	if err := ops.RemoveNetworkMount(context.Background(), "media"); err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+
+	var sawReset, sawRmUnit, sawRmdir, sawReload bool
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		switch {
+		case strings.Contains(j, "reset-failed -- "+autoName):
+			sawReset = true
+		case strings.Contains(j, "rm -f") && strings.Contains(j, autoName):
+			sawRmUnit = true
+		case strings.Contains(j, "rmdir") && strings.Contains(j, where):
+			sawRmdir = true
+		case strings.Contains(j, "daemon-reload"):
+			sawReload = true
+		}
+	}
+	if !sawReset {
+		t.Errorf("F2: a failed unit must be reset-failed on remove; calls: %v", rr.calls)
+	}
+	if !sawRmUnit {
+		t.Errorf("the unit files must be removed; calls: %v", rr.calls)
+	}
+	if !sawRmdir {
+		t.Errorf("F1: the empty mountpoint dir must be rmdir'd; calls: %v", rr.calls)
+	}
+	if !sawReload {
+		t.Errorf("daemon-reload must run after removal; calls: %v", rr.calls)
+	}
+}
+
+// rmdir is used (never rm -rf) — the fail-safe: a non-empty dir is left in place, not force-removed.
+func TestRemoveNetworkMount_NeverForceRemovesMountpoint(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	mountName, _ := UnitNameForMount(spec.Where())
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	_ = os.WriteFile(filepath.Join(unitDir, mountName), []byte(renderNetworkMountUnit(spec)), 0o644)
+	_ = os.WriteFile(filepath.Join(unitDir, autoName), []byte(renderNetworkAutomountUnit(spec)), 0o644)
+
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
+	if err := ops.RemoveNetworkMount(context.Background(), "media"); err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		if strings.Contains(j, "rm -rf") || (strings.Contains(j, "rm ") && strings.Contains(j, "/mnt/felhom-drives/media") && !strings.Contains(j, "rmdir")) {
+			t.Errorf("mountpoint cleanup must be rmdir-only (never rm -rf); offending call: %s", j)
+		}
+	}
+}