agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

cmd/felhom-agent/guesthook_bulletproof_test.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// F10/rc255 (CAMPAIGN-3): a guest-hook phase body that PANICS must never crash the process — the hook
+// must return cleanly so the guest start proceeds (a nonzero exit blocks the start). runHookPhase
+// recovers the panic and returns.
+func TestRunHookPhase_PanicRecovered(t *testing.T) {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		runHookPhase("9201", "pre-start", 5*time.Second, func(context.Context) {
+			panic("simulated heal panic (e.g. a future Heal bug)")
+		})
+	}()
+	select {
+	case <-done:
+		// returned cleanly — the guest start would proceed
+	case <-time.After(3 * time.Second):
+		t.Fatal("runHookPhase did not return after a panicking body (would have crashed the hook)")
+	}
+}
+
+// A phase body that overruns the timeout must be abandoned — the hook returns rather than hanging the
+// PVE start task. (The body's context is cancelled; the hook does not wait for the body to notice.)
+func TestRunHookPhase_TimeoutReturns(t *testing.T) {
+	bodyCtxCancelled := make(chan struct{}, 1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		runHookPhase("9201", "post-start", 20*time.Millisecond, func(ctx context.Context) {
+			<-ctx.Done() // simulate a body that respects cancellation eventually
+			bodyCtxCancelled <- struct{}{}
+		})
+	}()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("runHookPhase did not return after the timeout (would hang the guest start)")
+	}
+	select {
+	case <-bodyCtxCancelled:
+		// the body's context was cancelled at the deadline — the intended signal
+	case <-time.After(time.Second):
+		t.Fatal("the phase body's context was not cancelled at the timeout")
+	}
+}
+
+// A body that errors (returns normally, no panic) is fine — the hook returns cleanly.
+func TestRunHookPhase_NormalBodyReturns(t *testing.T) {
+	ran := false
+	runHookPhase("9201", "pre-start", time.Second, func(context.Context) { ran = true })
+	if !ran {
+		t.Fatal("the phase body must run")
+	}
+}

cmd/felhom-agent/main.go

@@ -44,6 +44,7 @@ import (
 	"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/selfheal"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
@@ -67,18 +68,46 @@ func runGuestHook(args []string) {
 	vmid, phase := args[0], args[1]
 	switch phase {
 	case guesthook.PhasePreStart:
-		created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
-		if len(created) > 0 {
-			fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
-		}
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
-		}
+		// F10/rc255 (CAMPAIGN-3): a pre-start hook that panics or hangs would BLOCK the guest start (the
+		// campaign saw a guest bricked while a unit sat start-limited). Every phase runs recover-wrapped
+		// under a hard timeout and this function ALWAYS returns cleanly (main then exits 0).
+		runHookPhase(vmid, "pre-start", 30*time.Second, func(context.Context) {
+			created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
+			if len(created) > 0 {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
+			}
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
+			}
+		})
 	case guesthook.PhasePostStart:
-		// Bounded: a hook must be fast; a wedged systemd call must not hold the start task.
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		postStartNetworkReassertFn(ctx, vmid)
+		runHookPhase(vmid, "post-start", 30*time.Second, func(ctx context.Context) {
+			postStartNetworkReassertFn(ctx, vmid)
+		})
+	}
+}
+
+// runHookPhase runs one guest-hook phase body under a hard timeout with a panic recover, so NO phase
+// can ever fail the guest start (CAMPAIGN-3 F10/rc255). A panic is logged to the PVE task log (stderr)
+// and swallowed; a body that overruns the timeout is abandoned (its context is cancelled) while the
+// hook returns. This is the Go belt; the wrapper script's `|| true; exit 0` is the shell belt.
+func runHookPhase(vmid, phase string, timeout time.Duration, body func(context.Context)) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	done := make(chan struct{})
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s PANIC recovered (boot continues): %v\n", vmid, phase, r)
+			}
+			close(done)
+		}()
+		body(ctx)
+	}()
+	select {
+	case <-done:
+	case <-ctx.Done():
+		fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s timed out after %s (boot continues)\n", vmid, phase, timeout)
 	}
 }
 
@@ -795,6 +824,16 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 		// bind that a re-provision dropped — before serving, so the drive is back in the guest config
 		// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
 		localSrv.ReassertGuestBinds(ctx)
+		// F12 (CAMPAIGN-3, CRITICAL): before re-arming, migrate any already-installed network-storage
+		// units to the current template — the units installed before 0.85 carry the network-online
+		// ordering cycle that makes each host boot a coin flip (networking lost on one boot, the automount
+		// on the next). A general template-drift reconcile (content-hash compare, batched daemon-reload),
+		// idempotent. Runs BEFORE the reassert sweep so the re-armed triggers are the fixed units.
+		if mountReasserter != nil {
+			if n := mountReasserter.MigrateNetworkUnits(ctx); n > 0 {
+				logger.Info("network-storage units migrated to the current template (F12 boot-ordering fix)", "migrated", n)
+			}
+		}
 		// RCA fix 1 (AUDIT-nas-cwa-rca-2026-07-11): re-arm idle NAS automount triggers ONCE at startup —
 		// covers the host-boot ordering where guests autostarted before the agent (their fresh namespaces
 		// missed the triggers). Deliberately NOT in the 20 s ticker: an idle trigger is healthy and must
@@ -822,6 +861,45 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 				}
 			}
 		}()
+		// CAMPAIGN-3 Part 6: node self-heal watchdog (appliance-gated). One heal — host networking
+		// (F12-class defense in depth: any boot leaving networking down is detected + remedied with the
+		// exact `systemctl start networking.service` the morning recovery ran by hand). On a byo host the
+		// check WARNs but the remedy is structurally unreachable (the Manager gates before any exec).
+		{
+			shMode := proxmox.RunnerMode(cfg.Privileged.Mode)
+			if shMode == "" {
+				shMode = proxmox.RunnerSudo
+			}
+			shRunner := &proxmox.ExecRunner{Mode: shMode, SudoPath: cfg.Privileged.SudoPath}
+			systemctlBin := cfg.Privileged.Systemctl
+			if systemctlBin == "" {
+				systemctlBin = "/usr/bin/systemctl"
+			}
+			shMgr := &selfheal.Manager{
+				Appliance: cfg.IsAppliance(),
+				Interval:  60 * time.Second,
+				Heals: []selfheal.Heal{&selfheal.NetworkingHeal{
+					IsActive: selfheal.SystemctlIsActive("networking.service"),
+					HasRoute: selfheal.HasDefaultRoute,
+					Start: func(ctx context.Context) error {
+						_, stderr, err := shRunner.Run(ctx, systemctlBin, "start", "networking.service")
+						if err != nil {
+							return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(stderr)))
+						}
+						return nil
+					},
+					Sleep:  selfheal.RealSleep,
+					Logger: logger,
+				}},
+				Logger: logger,
+			}
+			mode := "byo"
+			if cfg.IsAppliance() {
+				mode = "appliance"
+			}
+			logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
+			go shMgr.Watch(ctx)
+		}
 		// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
 		localSrv.RecoverFormatJob(ctx)
 		// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete

configs/felhom-agent.sudoers

@@ -143,11 +143,23 @@ Cmnd_Alias FELHOM_STALELOCK = \
 # `.automount` variants + the unit-file removal. The unit FILE name is the systemd-escaped mountpoint,
 # which always begins `mnt-felhom` (the mountpoint is /mnt/felhom-drives/<name>), so the rm glob is scoped
 # to felhom mount units only. mkdir of the mountpoint reuses FELHOM_INTERMEDIARY's /mnt/felhom-drives/*.
+# CAMPAIGN-3 additions (loud, per the no-widening rule):
+#  - `systemctl reset-failed -- mnt-felhom*`: F10 (CRITICAL) — a NAS automount that hit
+#    mount-start-limit-hit during an outage was re-armable by NO platform path; the reassert now
+#    reset-failed's the stuck unit before `enable --now` (which the start-limit otherwise refuses),
+#    and RemoveNetworkMount clears failed-state residue (F2). Scoped to felhom mount units (the unit
+#    name is the systemd-escaped mountpoint, always beginning `mnt-felhom`). reset-failed only clears
+#    a unit's failed latch — it cannot start/stop/alter anything.
+#  - `rmdir /mnt/felhom-drives/*`: F1 — remove the now-empty mountpoint dir a removed share leaves
+#    behind (the campaign accumulated 10 stub-shaped leftovers). rmdir ONLY (never rm -rf): it refuses
+#    a non-empty dir, so unexpected data is preserved, not destroyed — a fail-safe grant.
 Cmnd_Alias FELHOM_NETMOUNT = \
   /usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/units/* /etc/systemd/system/*.automount, \
   /usr/bin/systemctl enable --now -- *.automount, \
   /usr/bin/systemctl disable -- *.automount, \
   /usr/bin/systemctl stop -- *.automount, \
+  /usr/bin/systemctl reset-failed -- mnt-felhom*, \
+  /usr/bin/rmdir /mnt/felhom-drives/*, \
   /usr/bin/rm -f /etc/systemd/system/mnt-felhom*
 
 # Offsite WG tunnel (S3, doc 06 §3.3). The agent manages wg-quick@wg-felhom as an agent-managed
@@ -222,4 +234,13 @@ Cmnd_Alias FELHOM_OOB = \
   /usr/sbin/nft add element inet felhom_oob operator_ips *, \
   /usr/sbin/nft add element inet felhom_oob ssh_port *
 
-felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR
+# Node self-heal (CAMPAIGN-3 Part 6, F12-class defense in depth). The ONE fixed unit the appliance
+# watchdog may (re)start when a boot leaves networking down — the exact command the morning recovery
+# ran by hand after the F12 host loss. FIXED unit, no glob: this grant alone cannot harm — starting
+# networking.service is precisely what the boot should have done. The remedy is ALSO code-gated on
+# deployment_mode="appliance" (the Manager refuses to invoke it on a byo host); the sudoers grant is
+# the coarse floor, the mode gate is the fine one.
+Cmnd_Alias FELHOM_SELFHEAL = \
+  /usr/bin/systemctl start networking.service
+
+felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_SELFHEAL

internal/capability/manifest.go

@@ -73,6 +73,13 @@ var manifest = []Capability{
 	{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "felhom-x.mount"}, false},
 	{"mount-unit-stop", "mount unit stop", "/usr/bin/systemctl", []string{"stop", "--", "felhom-x.mount"}, false},
 
+	// ---- Network storage re-arm + cleanup (CAMPAIGN-3 F10/F1) ----
+	{"netmount-reset-failed", "NAS automount re-arm after start-limit (F10)", "/usr/bin/systemctl", []string{"reset-failed", "--", "mnt-felhom\\x2ddrives-media.automount"}, false},
+	{"netmount-rmdir", "removed-share mountpoint cleanup (F1)", "/usr/bin/rmdir", []string{"/mnt/felhom-drives/media"}, false},
+
+	// ---- Node self-heal (CAMPAIGN-3 F12-class, appliance-gated in code) ----
+	{"selfheal-networking-start", "appliance networking recovery at boot (F12 defense in depth)", "/usr/bin/systemctl", []string{"start", "networking.service"}, false},
+
 	// ---- Provisioning back-half ----
 	{"provision-chown", "bootstrap mount guest-root chown", "/usr/bin/chown", []string{"-R", "100000:100000", "/var/lib/felhom-agent/guests/9201"}, false},
 	{"provision-config-mount", "bootstrap config bind mount", "/usr/sbin/pct", []string{"set", "9201", "-mp0", "/var/lib/felhom-agent/guests/9201"}, false},

internal/config/config.go

@@ -36,11 +36,26 @@ type Config struct {
 	SelfUpdate  SelfUpdateConfig  `json:"selfupdate"`
 	LogLevel    string            `json:"log_level"` // debug|info|warn|error (default info)
 
+	// DeploymentMode gates host-service self-heal (CAMPAIGN-3 Part 6). "appliance" = a Felhom-managed
+	// node the agent may remediate (e.g. start networking at boot — F12-class defense in depth). Any
+	// other value, including absent/unknown, is treated as "byo" (a customer's own host): the self-heal
+	// CHECK still runs and WARNs, but the REMEDY is structurally unreachable. Fail-safe to byo — never
+	// touch a host we do not own. Distinct from Privileged.Mode (sudo vs direct exec) — do NOT overload.
+	DeploymentMode string `json:"deployment_mode,omitempty"`
+
 	// SourcePath is the file this config was loaded from ("" = all-env). Set by Load, never
 	// serialized — the pbsdr bridge's escrow.pbs_storage_id seed writes back to it.
 	SourcePath string `json:"-"`
 }
 
+// DeploymentModeAppliance is the ONLY value that unlocks host-service self-heal. Everything else,
+// including "" and any typo, is byo (fail-safe — a host we do not own is never remediated).
+const DeploymentModeAppliance = "appliance"
+
+// IsAppliance reports whether this node is a Felhom-managed appliance (self-heal remedies allowed).
+// Fail-safe: absent/unknown → false (byo).
+func (c *Config) IsAppliance() bool { return c.DeploymentMode == DeploymentModeAppliance }
+
 // OOBConfig configures the dedicated felhom-sshd OOB access instance + belt (TASK H1). **Enabled
 // DEFAULTS TO FALSE** — a rollout to a box without explicit oob.enabled=true is a no-op (no port
 // claim, no config render, no belt mutation, no oob report stanza) until the operator endpoint +
@@ -575,6 +590,9 @@ func applyEnv(cfg *Config) {
 	if v := os.Getenv("FELHOM_AGENT_LOG_LEVEL"); v != "" {
 		cfg.LogLevel = v
 	}
+	if v := os.Getenv("FELHOM_AGENT_DEPLOYMENT_MODE"); v != "" {
+		cfg.DeploymentMode = v
+	}
 	// hub
 	if v := os.Getenv("FELHOM_AGENT_HUB_URL"); v != "" {
 		cfg.Hub.URL = v

internal/config/config_test.go

@@ -133,3 +133,41 @@ func TestLoadFileThenEnvOverride(t *testing.T) {
 		t.Errorf("default endpoint lost: %q", cfg.Proxmox.Endpoint)
 	}
 }
+
+// CAMPAIGN-3 Part 6: deployment_mode gates node self-heal, and it is FAIL-SAFE to byo — absent or any
+// unknown value is byo, ONLY the exact "appliance" unlocks the remedy.
+func TestIsAppliance_FailSafeToByo(t *testing.T) {
+	cases := []struct {
+		mode string
+		want bool
+	}{
+		{"appliance", true},
+		{"byo", false},
+		{"", false},          // absent field → byo (fail-safe)
+		{"Appliance", false}, // case-sensitive — a typo must not unlock the remedy
+		{"garbage", false},
+	}
+	for _, c := range cases {
+		cfg := &Config{DeploymentMode: c.mode}
+		if got := cfg.IsAppliance(); got != c.want {
+			t.Errorf("IsAppliance(mode=%q) = %t, want %t", c.mode, got, c.want)
+		}
+	}
+}
+
+// The env overlay can set deployment_mode (FELHOM_AGENT_DEPLOYMENT_MODE).
+func TestDeploymentModeEnvOverlay(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "agent.json")
+	if err := os.WriteFile(path, []byte(`{"proxmox":{"node":"n","token":"u@pve!t=s"},"deployment_mode":"byo"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELHOM_AGENT_DEPLOYMENT_MODE", "appliance")
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if !cfg.IsAppliance() {
+		t.Errorf("env overlay did not set deployment_mode: %q", cfg.DeploymentMode)
+	}
+}

internal/guesthook/install.go

@@ -28,9 +28,14 @@ var SnippetPath = filepath.Join(SnippetDir, SnippetName)
 
 // snippetBody is the tiny wrapper PVE execs as `<script> <vmid> <phase>`. It delegates to the agent
 // binary so the heal LOGIC is the unit-tested Go, never duplicated (divergence-proof) shell. Executable.
+// The wrapper NEVER exec's and ALWAYS exits 0 (CAMPAIGN-3 F10/rc255 belt): a hook that exits nonzero
+// aborts the guest start. `exec` would surface the binary's exit code to PVE; instead we run it as a
+// child, swallow any nonzero (missing/crashed binary, OOM-kill), and `exit 0` unconditionally. The Go
+// side has its own recover + per-phase timeout — this is the second belt at the shell layer.
 const snippetBody = `#!/bin/sh
 # felhom-agent guest pre-start self-heal hook (C1 net). PVE calls: <script> <vmid> <phase>.
-exec ` + AgentBin + ` guest-hook "$1" "$2"
+` + AgentBin + ` guest-hook "$1" "$2" || true
+exit 0
 `
 
 // InstallSnippet writes the pre-start hook wrapper into the PVE snippets dir (idempotent, root-owned,

internal/guesthook/netreassert.go

@@ -42,8 +42,8 @@ func PostStartNetworkReassert(ctx context.Context, vmid string) {
 // real host surface).
 func postStartNetworkReassert(ctx context.Context, vmid string, ops netReasserter, sees func(ctx context.Context, vmid, path string) bool) {
 	for _, res := range ops.ReassertNetworkAutomounts(ctx) {
-		if res.Err != nil || res.Action == storage.NetReassertSkipNone {
-			continue // already reported by the ops logger / nothing expected in the guest
+		if !res.Remediates() {
+			continue // foreign/errored rows expect nothing in the guest (already logged by the ops layer)
 		}
 		if sees(ctx, vmid, res.Where) {
 			fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — network share %s visible in guest (%s)\n",

internal/guesthook/netreassert_test.go

@@ -18,12 +18,12 @@ func (f *fakeReasserter) ReassertNetworkAutomounts(context.Context) []storage.Ne
 }
 
 // The post-start core must run the reassert pass and verify guest visibility for every share the
-// pass acted on (or found actively mounted) — and never for skip-none/errored rows.
+// pass acted on (or found actively mounted) — and never for foreign/errored rows.
 func TestPostStartNetworkReassert_Core(t *testing.T) {
 	ops := &fakeReasserter{results: []storage.NetReassertResult{
 		{Name: "media", Where: "/mnt/felhom-drives/media", Action: storage.NetReassertRearmed},
 		{Name: "active", Where: "/mnt/felhom-drives/active", Action: storage.NetReassertSkipActive},
-		{Name: "gone", Where: "/mnt/felhom-drives/gone", Action: storage.NetReassertSkipNone},
+		{Name: "foreign", Where: "/mnt/felhom-drives/foreign", Action: storage.NetReassertSkipForeign},
 	}}
 	var verified []string
 	postStartNetworkReassert(context.Background(), "9201", ops, func(_ context.Context, vmid, path string) bool {

internal/localapi/netreassert.go

@@ -45,8 +45,8 @@ func (s *Server) ReassertNetworkMounts(ctx context.Context) {
 			continue
 		}
 		for _, res := range results {
-			if res.Action == storage.NetReassertSkipNone || res.Err != nil {
-				continue // nothing expected in the guest for these rows
+			if !res.Remediates() {
+				continue // foreign/errored rows expect nothing in the guest
 			}
 			if s.guestAttach.GuestSeesMount(ctx, vmid, res.Where) {
 				s.logger.Debug("netreassert: guest sees network share", "vmid", vmid, "name", res.Name, "where", res.Where)

internal/selfheal/selfheal.go

@@ -0,0 +1,161 @@
+// Package selfheal is a minimal check/remedy registry for node-level self-healing, gated hard on
+// appliance deployment mode (CAMPAIGN-3 Part 6).
+//
+// Only ONE heal ships now: host networking recovery — F12-class defense in depth. The F12 fix (the
+// automount template no longer creates the boot ordering cycle) is the CURE for that specific instance;
+// this watchdog is the belt for the CLASS: any boot that leaves networking down for ANY reason (a future
+// ordering bug, a flaky NIC bring-up, a botched netplan) is detected and — on a Felhom-managed appliance
+// only — remedied with the exact command the morning recovery ran by hand (`systemctl start
+// networking.service`). The registry shape is the deliverable so future appliance heals slot in behind
+// the SAME gate; resist growing a zoo of heals.
+//
+// THE GATE (never touch a host we do not own): a BYO host runs the CHECK and WARNs, but the Manager
+// refuses to call any Remediate before the appliance test — the remedy is structurally unreachable on
+// byo (unit-tested: byo + unhealthy → zero privileged invocations).
+package selfheal
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// Heal is one node-level check + remedy. Healthy reports the current state (and a human detail for the
+// unhealthy log); Remediate attempts recovery and returns a terminal error if it gives up.
+type Heal interface {
+	Name() string
+	Healthy(ctx context.Context) (ok bool, detail string)
+	Remediate(ctx context.Context) error
+}
+
+// Manager runs the registered heals at boot and on a periodic watchdog. The appliance gate lives HERE,
+// before any Remediate — a byo node never reaches a remedy.
+type Manager struct {
+	Appliance bool
+	Interval  time.Duration // watchdog cadence (≈60s); <=0 disables the periodic loop
+	Heals     []Heal
+	Logger    *slog.Logger
+}
+
+// RunOnce evaluates every heal once (the boot pass). For each unhealthy heal it WARNs (both facts in
+// the detail); on an appliance it then remediates, on byo it stops at the WARN — the remedy exec is
+// never reached off-appliance.
+func (m *Manager) RunOnce(ctx context.Context) {
+	for _, h := range m.Heals {
+		ok, detail := h.Healthy(ctx)
+		if ok {
+			m.Logger.Debug("selfheal: node check healthy", "heal", h.Name())
+			continue
+		}
+		m.Logger.Warn("selfheal: node check FAILED", "heal", h.Name(), "detail", detail, "mode", m.mode())
+		if !m.Appliance {
+			// GATE: byo hosts are the customer's to manage — never remediate. (Red-proof: byo +
+			// unhealthy must record ZERO privileged invocations; the remedy is unreachable here.)
+			m.Logger.Warn("selfheal: byo host — remedy skipped (not ours to touch)", "heal", h.Name())
+			continue
+		}
+		if err := h.Remediate(ctx); err != nil {
+			m.Logger.Error("selfheal: remedy GAVE UP — host needs attention", "heal", h.Name(), "err", err)
+		}
+	}
+}
+
+// Watch runs RunOnce once immediately (the boot pass) then on every Interval tick until ctx is done.
+func (m *Manager) Watch(ctx context.Context) {
+	m.RunOnce(ctx)
+	if m.Interval <= 0 {
+		return
+	}
+	t := time.NewTicker(m.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			m.RunOnce(ctx)
+		}
+	}
+}
+
+func (m *Manager) mode() string {
+	if m.Appliance {
+		return "appliance"
+	}
+	return "byo"
+}
+
+// NetworkingHeal is the one shipped heal: host networking (F12-class). Healthy ⇔ networking.service is
+// active AND a default route exists. All three probes/actions are seams (unit-tested with fakes); Start
+// is the ONLY privileged one (`systemctl start networking.service`, appliance-gated by the Manager).
+type NetworkingHeal struct {
+	IsActive func(ctx context.Context) bool  // `systemctl is-active networking.service` (unprivileged)
+	HasRoute func(ctx context.Context) bool  // a default route exists (unprivileged)
+	Start    func(ctx context.Context) error // PRIVILEGED `systemctl start networking.service`
+	Sleep    func(ctx context.Context, d time.Duration)
+	Logger   *slog.Logger
+}
+
+func (n *NetworkingHeal) Name() string { return "networking" }
+
+// Healthy reports both facts so the WARN detail names exactly what is down.
+func (n *NetworkingHeal) Healthy(ctx context.Context) (bool, string) {
+	active := n.IsActive(ctx)
+	route := n.HasRoute(ctx)
+	if active && route {
+		return true, ""
+	}
+	return false, fmt.Sprintf("networking.service active=%t, default route present=%t", active, route)
+}
+
+// networkingBackoffs is the wait after each start attempt before re-checking — 10s/30s/60s gives a slow
+// NIC/DHCP time to come up. Package var so tests can shrink it.
+var networkingBackoffs = []time.Duration{10 * time.Second, 30 * time.Second, 60 * time.Second}
+
+// Remediate runs up to 3 `systemctl start networking.service` attempts, each followed by its backoff and
+// a health re-check. ERROR per failed attempt; INFO on recovery; a terminal give-up error if all fail.
+func (n *NetworkingHeal) Remediate(ctx context.Context) error {
+	for attempt := 1; attempt <= len(networkingBackoffs); attempt++ {
+		if err := n.Start(ctx); err != nil {
+			n.Logger.Error("node self-heal: networking start attempt failed", "attempt", attempt, "err", err)
+		}
+		n.Sleep(ctx, networkingBackoffs[attempt-1])
+		if ok, _ := n.Healthy(ctx); ok {
+			n.Logger.Info("node self-heal: networking recovered", "attempt", attempt)
+			return nil
+		}
+		n.Logger.Error("node self-heal: networking still down after start attempt", "attempt", attempt)
+	}
+	return fmt.Errorf("networking still down after %d attempts (host needs physical/console attention)", len(networkingBackoffs))
+}
+
+// --- production seams ------------------------------------------------------------------------------
+
+// SystemctlIsActive is the default IsActive: `systemctl is-active <unit>` prints "active" when up.
+// Unprivileged (unit state is world-readable) — NOT routed through sudo.
+func SystemctlIsActive(unit string) func(context.Context) bool {
+	return func(ctx context.Context) bool {
+		out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "--", unit).Output()
+		return strings.TrimSpace(string(out)) == "active"
+	}
+}
+
+// HasDefaultRoute is the default HasRoute: `ip route show default` is non-empty when a default route
+// exists. Unprivileged read.
+func HasDefaultRoute(ctx context.Context) bool {
+	out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output()
+	return err == nil && strings.TrimSpace(string(out)) != ""
+}
+
+// RealSleep is the default Sleep: a context-cancellable time.Sleep.
+func RealSleep(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}

internal/selfheal/selfheal_test.go

@@ -0,0 +1,134 @@
+package selfheal
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+)
+
+func quietLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError + 1}))
+}
+
+// fakeHeal is a controllable Heal: healthy toggles, and it counts remediation invocations.
+type fakeHeal struct {
+	healthy    bool
+	remediated int
+}
+
+func (f *fakeHeal) Name() string { return "fake" }
+func (f *fakeHeal) Healthy(context.Context) (bool, string) {
+	return f.healthy, "fake detail"
+}
+func (f *fakeHeal) Remediate(context.Context) error {
+	f.remediated++
+	f.healthy = true
+	return nil
+}
+
+// THE BYO GATE (red-proof): a byo Manager must NEVER reach a remedy, even with an unhealthy heal.
+func TestManager_ByoNeverRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: false, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 0 {
+		t.Fatalf("byo host must NEVER remediate, got %d invocations", h.remediated)
+	}
+}
+
+// An appliance Manager remediates an unhealthy heal.
+func TestManager_ApplianceRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: true, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 1 {
+		t.Fatalf("appliance must remediate an unhealthy heal once, got %d", h.remediated)
+	}
+}
+
+// A healthy heal is never remediated (either mode).
+func TestManager_HealthyNoRemediate(t *testing.T) {
+	for _, appliance := range []bool{true, false} {
+		h := &fakeHeal{healthy: true}
+		m := &Manager{Appliance: appliance, Heals: []Heal{h}, Logger: quietLogger()}
+		m.RunOnce(context.Background())
+		if h.remediated != 0 {
+			t.Fatalf("appliance=%t: a healthy heal must not be remediated, got %d", appliance, h.remediated)
+		}
+	}
+}
+
+// THE BYO GATE at the networking layer: force the networking check unhealthy under byo and assert the
+// privileged Start is invoked ZERO times (the gate is before any exec).
+func TestNetworkingHeal_ByoZeroPrivilegedInvocations(t *testing.T) {
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false },
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	m := &Manager{Appliance: false, Heals: []Heal{n}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if starts != 0 {
+		t.Fatalf("byo networking heal must invoke `systemctl start` ZERO times, got %d", starts)
+	}
+}
+
+// The remedy state machine: recovers on the 2nd attempt (start brings it up) → returns nil, one INFO.
+func TestNetworkingHeal_RecoversMidAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	active := false
+	attempts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return active },
+		HasRoute: func(context.Context) bool { return true },
+		Start: func(context.Context) error {
+			attempts++
+			if attempts >= 2 {
+				active = true // the 2nd start brings networking up
+			}
+			return nil
+		},
+		Sleep:  func(context.Context, time.Duration) {},
+		Logger: quietLogger(),
+	}
+	if err := n.Remediate(context.Background()); err != nil {
+		t.Fatalf("remedy should recover by attempt 2, got err %v", err)
+	}
+	if attempts != 2 {
+		t.Fatalf("recovery should have taken 2 start attempts, got %d", attempts)
+	}
+}
+
+// The remedy gives up after all attempts if networking never comes up — a terminal error, logged as
+// give-up by the Manager.
+func TestNetworkingHeal_GivesUpAfterAllAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false }, // never recovers
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	err := n.Remediate(context.Background())
+	if err == nil {
+		t.Fatal("remedy must return a terminal error when networking never recovers")
+	}
+	if starts != len(networkingBackoffs) {
+		t.Fatalf("remedy must try exactly %d times, got %d", len(networkingBackoffs), starts)
+	}
+}
+
+// withFastBackoffs swaps the real 10/30/60s backoffs for near-zero waits (the Sleep seam is faked
+// anyway, but keep the count/shape). Returns a restore func.
+func withFastBackoffs() func() {
+	orig := networkingBackoffs
+	networkingBackoffs = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	return func() { networkingBackoffs = orig }
+}

internal/storage/hostops.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -98,12 +99,12 @@ type MountSpec struct {
 // Binaries holds the absolute paths of the allow-listed binaries (overridable from config
 // so the sudoers entries and the agent agree on exact paths).
 type Binaries struct {
-	Systemctl string
-	Install   string
-	Smartctl  string
-	Lvs       string
-	Blkid     string // device signature probe (8C data-bearing detection)
-	Lsblk     string // partition/mount topology (8C)
+	Systemctl   string
+	Install     string
+	Smartctl    string
+	Lvs         string
+	Blkid       string // device signature probe (8C data-bearing detection)
+	Lsblk       string // partition/mount topology (8C)
 	MkfsExt4    string // 8C format executor (ext4) — now invoked by the guarded wrapper, not the agent directly
 	MkfsXfs     string // 8C format executor (xfs) — now invoked by the guarded wrapper, not the agent directly
 	MkfsGuarded string // Impl-1 Part B: the guarded-mkfs wrapper the agent execs (device+fstype)
@@ -155,18 +156,22 @@ func (b Binaries) withDefaults() Binaries {
 type SudoHostOps struct {
 	runner   proxmox.Runner
 	bins     Binaries
-	unitDir  string // where enabled units live (e.g. /etc/systemd/system)
-	stageDir string // agent-owned staging dir for unit files before install
+	unitDir  string     // where enabled units live (e.g. /etc/systemd/system)
+	stageDir string     // agent-owned staging dir for unit files before install
 	host     HostReader // root-free reads (mount table) for the Impl-1 Format claim guard
 	logger   *slog.Logger
+	// unitFailed reports whether a systemd unit is in the failed state (incl. start-limit-hit). An
+	// UNPRIVILEGED read (`systemctl is-failed`) — seam-injected so the reassert's F10 reset-failed path
+	// is unit-testable without a real systemd. Default set in NewSudoHostOps.
+	unitFailed func(ctx context.Context, unit string) bool
 }
 
 // SudoHostOpsConfig configures a SudoHostOps.
 type SudoHostOpsConfig struct {
 	Runner   proxmox.Runner
 	Bins     Binaries
-	UnitDir  string // default /etc/systemd/system
-	StageDir string // default <dataDir>/units; must be agent-writable
+	UnitDir  string     // default /etc/systemd/system
+	StageDir string     // default <dataDir>/units; must be agent-writable
 	Host     HostReader // default NewProcHostReader(); the Format claim guard's mount-table read
 	Logger   *slog.Logger
 }
@@ -190,15 +195,24 @@ func NewSudoHostOps(cfg SudoHostOpsConfig) *SudoHostOps {
 		host = NewProcHostReader()
 	}
 	return &SudoHostOps{
-		runner:   cfg.Runner,
-		bins:     cfg.Bins.withDefaults(),
-		unitDir:  unitDir,
-		stageDir: stageDir,
-		host:     host,
-		logger:   logger,
+		runner:     cfg.Runner,
+		bins:       cfg.Bins.withDefaults(),
+		unitDir:    unitDir,
+		stageDir:   stageDir,
+		host:       host,
+		logger:     logger,
+		unitFailed: systemctlIsFailed,
 	}
 }
 
+// systemctlIsFailed is the production unitFailed: `systemctl is-failed <unit>` prints "failed" for a
+// failed/start-limit-hit unit and exits nonzero otherwise. UNPRIVILEGED (unit state is world-readable)
+// — deliberately NOT routed through the sudo runner, so it needs no sudoers grant.
+func systemctlIsFailed(ctx context.Context, unit string) bool {
+	out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", "--", unit).Output()
+	return strings.TrimSpace(string(out)) == "failed"
+}
+
 // EnsureMount validates, renders, stages, installs and enables the .mount unit.
 func (h *SudoHostOps) EnsureMount(ctx context.Context, spec MountSpec) error {
 	// VALIDATE FIRST — refuse before constructing any command.

internal/storage/netmigrate_test.go

@@ -0,0 +1,170 @@
+package storage
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// F12 (CAMPAIGN-3, CRITICAL): NEITHER rendered network-storage unit may carry a network-online.target
+// ordering — that is exactly what closed the boot ordering cycle. The .mount keeps `_netdev` (the
+// correct, sufficient network ordering for the real mount). Companion red-proof: re-adding either
+// `After=`/`Wants=network-online.target` line to a template makes these assertions fail.
+func TestRenderNetworkUnits_NoNetworkOnlineOrdering(t *testing.T) {
+	for _, spec := range []NetworkMountSpec{
+		{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000},
+		{Name: "docs", Protocol: ProtocolSMB, Server: "10.0.0.6", Export: "share", CredsRef: "/var/lib/felhom-agent/smb-creds/docs.cred", MappedUID: 1000, MappedGID: 1000},
+	} {
+		mountUnit := renderNetworkMountUnit(spec)
+		autoUnit := renderNetworkAutomountUnit(spec)
+		if strings.Contains(mountUnit, "network-online.target") {
+			t.Errorf("[%s] .mount unit still orders network-online.target (F12 cycle):\n%s", spec.Name, mountUnit)
+		}
+		if strings.Contains(autoUnit, "network-online.target") {
+			t.Errorf("[%s] .automount unit still orders network-online.target (F12 cycle):\n%s", spec.Name, autoUnit)
+		}
+		if !strings.Contains(mountUnit, "_netdev") {
+			t.Errorf("[%s] .mount unit lost _netdev — the ONLY correct network ordering for the real mount:\n%s", spec.Name, mountUnit)
+		}
+	}
+}
+
+// specFromNetworkUnits must reconstruct a spec that re-renders EXACTLY the installed unit pair — the
+// idempotency contract of the drift reconcile. A round-trip that isn't byte-exact would make
+// MigrateNetworkUnits rewrite on every pass (never converging).
+func TestSpecFromNetworkUnits_RoundTrips(t *testing.T) {
+	for _, spec := range []NetworkMountSpec{
+		{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 60},
+		{Name: "docs", Protocol: ProtocolSMB, Server: "10.0.0.6", Export: "share", CredsRef: "/var/lib/felhom-agent/smb-creds/docs.cred", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 120},
+	} {
+		mountUnit := renderNetworkMountUnit(spec)
+		autoUnit := renderNetworkAutomountUnit(spec)
+		got, ok := specFromNetworkUnits(mountUnit, autoUnit)
+		if !ok {
+			t.Fatalf("[%s] specFromNetworkUnits failed to parse its own render", spec.Name)
+		}
+		if renderNetworkMountUnit(got) != mountUnit {
+			t.Errorf("[%s] .mount round-trip mismatch:\nWANT:\n%s\nGOT:\n%s", spec.Name, mountUnit, renderNetworkMountUnit(got))
+		}
+		if renderNetworkAutomountUnit(got) != autoUnit {
+			t.Errorf("[%s] .automount round-trip mismatch:\nWANT:\n%s\nGOT:\n%s", spec.Name, autoUnit, renderNetworkAutomountUnit(got))
+		}
+	}
+}
+
+// legacyNetworkUnitPair renders the pre-0.85 units WITH the F12 network-online ordering, the exact
+// drift the migration must repair.
+func legacyNetworkUnitPair(spec NetworkMountSpec) (mount, auto string) {
+	mount = renderNetworkMountUnit(spec)
+	mount = strings.Replace(mount, "\n[Mount]\n", "\nAfter=network-online.target\nWants=network-online.target\n[Mount]\n", 1)
+	auto = renderNetworkAutomountUnit(spec)
+	auto = strings.Replace(auto, "\n[Automount]\n", "\nAfter=network-online.target\nWants=network-online.target\n[Automount]\n", 1)
+	return mount, auto
+}
+
+// MigrateNetworkUnits rewrites a drifted (legacy, network-online-carrying) unit pair exactly ONCE,
+// batches a single daemon-reload, and is idempotent (a second pass rewrites nothing). A clean unit is
+// left untouched.
+func TestMigrateNetworkUnits_RewritesDriftedOnceIdempotent(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	stageDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000, IdleTimeoutSec: 60}
+	mountName, err := UnitNameForMount(spec.Where())
+	if err != nil {
+		t.Fatalf("unit name: %v", err)
+	}
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	legacyMount, legacyAuto := legacyNetworkUnitPair(spec)
+	writeFile(t, filepath.Join(unitDir, mountName), legacyMount)
+	writeFile(t, filepath.Join(unitDir, autoName), legacyAuto)
+
+	rr := &recordingRunner{}
+	// installUnit stages then `install`s (recorded, not executed); rewrite the unit dir copy ourselves
+	// so the on-disk content reflects the migration for the idempotency re-read.
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: stageDir, Host: &fakeHostReader{}, Logger: quietLogger()})
+
+	migrated := ops.MigrateNetworkUnits(context.Background())
+	if migrated != 1 {
+		t.Fatalf("first pass must migrate exactly 1 unit, got %d", migrated)
+	}
+	// The recorded calls must include exactly one daemon-reload (batched), plus the two installs.
+	reloads, installs := 0, 0
+	for _, c := range rr.calls {
+		joined := strings.Join(c, " ")
+		if strings.Contains(joined, "daemon-reload") {
+			reloads++
+		}
+		if strings.Contains(joined, "install") {
+			installs++
+		}
+	}
+	if reloads != 1 {
+		t.Errorf("migration must batch exactly ONE daemon-reload, got %d (calls: %v)", reloads, rr.calls)
+	}
+	if installs != 2 {
+		t.Errorf("migration must rewrite both units (2 installs), got %d", installs)
+	}
+	// The staged content the installer would have placed must be the CLEAN template. Simulate the
+	// install landing (installUnit staged to stageDir/<unit>), then re-read for idempotency.
+	applyStaged(t, stageDir, unitDir, mountName)
+	applyStaged(t, stageDir, unitDir, autoName)
+	if got := readFile(t, filepath.Join(unitDir, mountName)); strings.Contains(got, "network-online.target") {
+		t.Errorf("migrated .mount still carries network-online.target:\n%s", got)
+	}
+
+	rr.calls = nil
+	if migrated := ops.MigrateNetworkUnits(context.Background()); migrated != 0 {
+		t.Fatalf("second pass over already-current units must migrate 0, got %d", migrated)
+	}
+	if len(rr.calls) != 0 {
+		t.Errorf("idempotent second pass must construct ZERO commands, got: %v", rr.calls)
+	}
+}
+
+// A non-marker unit file in the unit dir is never touched by the migration.
+func TestMigrateNetworkUnits_IgnoresForeignUnits(t *testing.T) {
+	unitDir := t.TempDir()
+	writeFile(t, filepath.Join(unitDir, "some-service.mount"), "[Unit]\nDescription=not ours\n[Mount]\nWhere=/x\n")
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
+	if migrated := ops.MigrateNetworkUnits(context.Background()); migrated != 0 {
+		t.Fatalf("a foreign unit must not be migrated, got %d", migrated)
+	}
+	if len(rr.calls) != 0 {
+		t.Errorf("a foreign unit must construct zero commands, got: %v", rr.calls)
+	}
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(b)
+}
+
+// applyStaged mirrors what the (recorded, not executed) `install` would do: copy the agent-staged unit
+// into the unit dir, so the idempotency re-read sees the migrated content.
+func applyStaged(t *testing.T, stageDir, unitDir, unitName string) {
+	t.Helper()
+	src := filepath.Join(stageDir, unitName)
+	b, err := os.ReadFile(src)
+	if err != nil {
+		return // installUnit stages before the recorded install; if absent, nothing to mirror
+	}
+	writeFile(t, filepath.Join(unitDir, unitName), string(b))
+}

internal/storage/netmount.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"os"
@@ -246,8 +248,12 @@ func renderNetworkMountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): NO network-online.target ordering here. `_netdev` in Options is the
+	// correct + sufficient network ordering for the REAL mount — systemd classes a _netdev mount under
+	// remote-fs.target and orders it after the network without a hand-written After/Wants. A literal
+	// `After=network-online.target` on this unit (which the .automount pulls in via local-fs) closed the
+	// boot ordering cycle networking→local-fs→automount→network-online→networking; systemd broke it by
+	// DELETING an arbitrary job (one boot lost networking entirely, the next lost the automount).
 	b.WriteString("\n[Mount]\n")
 	fmt.Fprintf(&b, "What=%s\n", s.mountSource())
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
@@ -263,8 +269,12 @@ func renderNetworkAutomountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage automount %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): the automount unit gets NO network relation of ANY kind. A trigger
+	// needs no network — it just watches the mountpoint and fires the .mount on first access (the
+	// .mount's `_netdev` then orders the real mount after the network). An automount is implicitly
+	// ordered Before=local-fs.target; adding After/Wants=network-online.target here created the boot
+	// ordering cycle that cost the host its network on one boot and its NAS on the next. Keep this unit
+	// orderable before local-fs WITHOUT dragging the network into that transaction.
 	b.WriteString("\n[Automount]\n")
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
 	fmt.Fprintf(&b, "TimeoutIdleSec=%d\n", s.idleTimeout())
@@ -347,6 +357,10 @@ func (h *SudoHostOps) EnsureNetworkMount(ctx context.Context, spec NetworkMountS
 	if err := ValidateNetworkMountSpec(spec); err != nil {
 		return err
 	}
+	// Template-drift reconcile: bring any already-installed units up to the current template before we
+	// touch the unit dir (F12 — a pre-0.85 unit still carrying the network-online ordering gets rewritten
+	// here even if the daemon-startup migration hasn't run in this process). Best-effort; never blocks add.
+	h.MigrateNetworkUnits(ctx)
 	// Defense in depth: never realise a network mount outside the user-data namespace.
 	if NetworkMountRole(spec.Where()) != RoleUserData {
 		return fmt.Errorf("netmount: refusing to mount outside the user-data namespace: %s", spec.Where())
@@ -427,6 +441,10 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 		h.logger.Debug("netmount: remove step", "verb", step[0], "unit", step[len(step)-1],
 			"ok", err == nil) // a "not loaded" failure here is expected + tolerated
 	}
+	// F2 (CAMPAIGN-3): clear any failed/start-limit runtime state on the pair BEFORE the files go, or
+	// systemd keeps them as `not-found failed` residue after daemon-reload. reset-failed while the units
+	// are still loaded; tolerate the not-failed case (nothing to reset).
+	h.resetNetworkUnitsIfFailed(ctx, automountUnit, mountUnit)
 
 	destAuto := filepath.Join(h.unitDir, automountUnit)
 	destMount := filepath.Join(h.unitDir, mountUnit)
@@ -441,10 +459,184 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 	if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
 		return fmt.Errorf("netmount: daemon-reload: %w", err)
 	}
+	// F1 (CAMPAIGN-3): remove the now-empty mountpoint directory (the campaign accumulated 10 stub-shaped
+	// leftovers). rmdir ONLY — a non-empty dir (unexpected data present) is left in place with a WARN, the
+	// fail-safe; `rm -rf` is forbidden here. The host removal propagates into running guests through the
+	// shared bind; a fresh guest re-binds cleanly on next start.
+	h.rmdirMountpoint(ctx, where)
 	h.logger.Info("netmount: removed network mount", "name", name, "where", where)
 	return nil
 }
 
+// resetNetworkUnitsIfFailed reset-failed's any of the given units that is in the failed state (F2 —
+// leave no `not-found failed`/start-limit residue behind a remove or a rolled-back add). Unprivileged
+// is-failed read + the FELHOM_NETMOUNT reset-failed grant; every step tolerated.
+func (h *SudoHostOps) resetNetworkUnitsIfFailed(ctx context.Context, units ...string) {
+	for _, unit := range units {
+		if h.unitFailed == nil || !h.unitFailed(ctx, unit) {
+			continue
+		}
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			h.logger.Warn("netmount: reset-failed tolerated failure", "unit", unit, "err", err)
+		}
+	}
+}
+
+// rmdirMountpoint removes an empty network mountpoint dir under NetworkMountRoot. rmdir refuses a
+// non-empty dir (the fail-safe): unexpected data is preserved and flagged, never rm -rf'd. Best-effort.
+func (h *SudoHostOps) rmdirMountpoint(ctx context.Context, where string) {
+	if !strings.HasPrefix(where, NetworkMountRoot+"/") {
+		return // defense in depth: only ever under the bind root
+	}
+	if err := h.run(ctx, "/usr/bin/rmdir", where); err != nil {
+		// rmdir fails on a non-empty dir — leave it (fail-safe) and flag it for the operator.
+		h.logger.Warn("netmount: mountpoint dir not removed (non-empty or busy — left in place, fail-safe)",
+			"where", where, "err", err)
+	}
+}
+
+// MigrateNetworkUnits reconciles every marker-owned network-storage unit file on disk against a fresh
+// render of its own reconstructed spec — a general template-drift reconcile (the git-sync pattern:
+// content-hash compare, rewrite on drift, batched daemon-reload). It exists because a template change
+// must reach ALREADY-INSTALLED units, not only future adds: the F12 fix (CAMPAIGN-3) removed the
+// network-online ordering that turned every host boot with an enrolled share into a coin flip, and the
+// units installed before 0.85 still carry the ordering cycle until they are rewritten. Runs at agent
+// startup (before the reassert sweep) and at the head of EnsureNetworkMount. Idempotent: a unit already
+// byte-identical to its fresh render is left untouched (second pass rewrites nothing). Best-effort per
+// unit; one INFO line per migrated unit. Returns the count migrated.
+func (h *SudoHostOps) MigrateNetworkUnits(ctx context.Context) int {
+	entries, err := os.ReadDir(h.unitDir)
+	if err != nil {
+		h.logger.Warn("netmigrate: reading unit dir failed", "err", err)
+		return 0
+	}
+	migrated := 0
+	changed := false
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".mount") {
+			continue // the .mount carries What/Type; the paired .automount mirrors Where
+		}
+		mountPath := filepath.Join(h.unitDir, e.Name())
+		mountData, rerr := os.ReadFile(mountPath)
+		if rerr != nil || !strings.Contains(string(mountData), netUnitMarker) {
+			continue // unreadable or not one of ours
+		}
+		automountName := strings.TrimSuffix(e.Name(), ".mount") + ".automount"
+		automountData, aerr := os.ReadFile(filepath.Join(h.unitDir, automountName))
+		if aerr != nil {
+			continue // a .mount with no paired .automount is malformed — not ours to guess
+		}
+		spec, ok := specFromNetworkUnits(string(mountData), string(automountData))
+		if !ok {
+			continue
+		}
+		freshMount := renderNetworkMountUnit(spec)
+		freshAuto := renderNetworkAutomountUnit(spec)
+		if contentHash(string(mountData)) == contentHash(freshMount) &&
+			contentHash(string(automountData)) == contentHash(freshAuto) {
+			continue // already current — the idempotent no-op
+		}
+		if err := h.installUnit(ctx, e.Name(), freshMount); err != nil {
+			h.logger.Warn("netmigrate: rewriting mount unit failed", "unit", e.Name(), "err", err)
+			continue
+		}
+		if err := h.installUnit(ctx, automountName, freshAuto); err != nil {
+			h.logger.Warn("netmigrate: rewriting automount unit failed", "unit", automountName, "err", err)
+			continue
+		}
+		changed = true
+		migrated++
+		h.logger.Info("netmigrate: migrated network-storage unit to the current template (F12: dropped the boot ordering cycle)",
+			"name", spec.Name, "where", spec.Where())
+	}
+	if changed {
+		if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
+			h.logger.Warn("netmigrate: daemon-reload after migration failed", "err", err)
+		}
+	}
+	return migrated
+}
+
+// contentHash is the SHA-256 hex of a unit file's content — the drift comparator (git-sync pattern).
+func contentHash(s string) string {
+	sum := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(sum[:])
+}
+
+// specFromNetworkUnits reconstructs the NetworkMountSpec that renders EXACTLY the given installed unit
+// pair — the fresh-render input for the drift reconcile. Round-trips by construction: every field the
+// render templates read is recovered (proto/server/export/where from the .mount; the SMB uid/gid+creds
+// from its Options; the idle window from the .automount). A NFS spec's mapped uid never appears in a
+// rendered unit, so it is irrelevant to the render and left at the container default. ok=false for a
+// non-marker or unparseable pair.
+func specFromNetworkUnits(mountContent, automountContent string) (NetworkMountSpec, bool) {
+	proto, server, export, where, ok := parseNetworkMountUnit(mountContent)
+	if !ok {
+		return NetworkMountSpec{}, false
+	}
+	spec := NetworkMountSpec{
+		Name:      strings.TrimPrefix(where, NetworkMountRoot+"/"),
+		Protocol:  NetworkProtocol(proto),
+		Server:    server,
+		Export:    export,
+		MappedUID: 1000, // container default; unused by the NFS render, overwritten below for SMB
+		MappedGID: 1000,
+	}
+	if spec.Protocol == ProtocolSMB {
+		opts := unitLineValue(mountContent, "Options=")
+		if uid, ok := csvIntField(opts, "uid="); ok {
+			spec.MappedUID = uid - lxcUIDOffset
+		}
+		if gid, ok := csvIntField(opts, "gid="); ok {
+			spec.MappedGID = gid - lxcUIDOffset
+		}
+		spec.CredsRef = csvField(opts, "credentials=")
+	}
+	if idle, ok := csvIntField(unitLineValue(automountContent, "TimeoutIdleSec="), ""); ok && idle > 0 {
+		spec.IdleTimeoutSec = idle
+	}
+	return spec, true
+}
+
+// unitLineValue returns the value after the first line beginning with prefix (e.g. "Options="), trimmed.
+func unitLineValue(content, prefix string) string {
+	for _, line := range strings.Split(content, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, prefix) {
+			return strings.TrimPrefix(line, prefix)
+		}
+	}
+	return ""
+}
+
+// csvField finds the comma-separated token with the given key prefix (e.g. "credentials=") and returns
+// its value. "" if absent.
+func csvField(csv, key string) string {
+	for _, tok := range strings.Split(csv, ",") {
+		if strings.HasPrefix(tok, key) {
+			return strings.TrimPrefix(tok, key)
+		}
+	}
+	return ""
+}
+
+// csvIntField parses the int value of a comma-separated key (e.g. "uid=") — or, when key is "", parses
+// the whole string as an int (for a bare value like TimeoutIdleSec's already-extracted number).
+func csvIntField(csv, key string) (int, bool) {
+	val := csv
+	if key != "" {
+		val = csvField(csv, key)
+	}
+	if val == "" {
+		return 0, false
+	}
+	n, err := strconv.Atoi(strings.TrimSpace(val))
+	if err != nil {
+		return 0, false
+	}
+	return n, true
+}
+
 // ListNetworkMounts enumerates the installed network-storage units and reports per-share liveness. It
 // reads the (world-readable) unit dir + /proc/mounts and TCP-probes each NAS endpoint with a short
 // timeout — it NEVER stat()s the (possibly EIO/D-state) mountpoint, so a black-holed NAS cannot wedge

internal/storage/netreassert.go

@@ -6,29 +6,43 @@ import (
 	"strings"
 )
 
-// Network-mount guest-reboot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1).
+// Network-mount guest-reboot / boot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1, hardened by
+// CAMPAIGN-3 F9/F10/F11).
 //
-// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or
-// an actively-mounted nfs4) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
+// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or an
+// actively-mounted nfs4/cifs) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
 // share silently degrades to a local stub directory inside the guest. The heal is host-side and
-// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates
-// live into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
+// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates live
+// into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
 //
-// The action uses the sudoers-granted verbs only (`systemctl stop -- *.automount` +
-// `systemctl enable --now -- *.automount`; there is NO restart grant). Idempotent: re-arming an
-// already-armed trigger just recreates it — same end state, and the fresh mount event is harmless.
-// An ACTIVE real mount is never touched (stopping the automount of a live mount would churn it).
+// CAMPAIGN-3 hardening:
+//   - F11 (read the right unit): the decision is driven ONLY by the host `/proc/mounts` fstype AT the
+//     mountpoint — an ACTIVE real mount (nfs4/cifs) is inherited and left alone; anything else is
+//     re-armed. The `.automount` unit's own state is NEVER consulted (an armed trigger always reports
+//     "active", which is exactly why a state-of-the-automount check mis-skips idle triggers).
+//   - F10 (re-arm for real): a `.mount`/`.automount` left in `failed`/start-limit-hit state (the
+//     campaign's unexport→idle-timeout→access×5 sequence) is `reset-failed` FIRST — without it the
+//     `enable --now` below is refused by the start limit and the share stays dead across every boot.
+//   - F9 (say what you did): the pass enumerates by the marker-owned unit files on disk (not by
+//     enablement or runtime state) and logs an INFO verdict line for EVERY share — an empty-looking
+//     sweep over N shares is structurally impossible.
+//
+// The action uses the sudoers-granted verbs only (`systemctl reset-failed -- *`, `systemctl stop -- *`,
+// `systemctl enable --now -- *`). Idempotent: re-arming an already-armed trigger recreates it — same end
+// state, and the fresh mount event is harmless. An ACTIVE real mount is never touched.
 
-// Reassert actions (the §8 decision table, encoded).
+// Reassert actions (the §8 decision table, encoded — CAMPAIGN-3 F11).
 const (
 	// NetReassertRearmed: the trigger was re-created (stop + enable --now) — the propagation heal.
 	NetReassertRearmed = "rearmed"
-	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh
-	// namespaces, nothing to do (verify only).
+	// NetReassertResetRearmed: the unit was failed/start-limited, reset-failed, THEN re-armed (F10).
+	NetReassertResetRearmed = "reset-failed+rearmed"
+	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh namespaces,
+	// nothing to do (verify only).
 	NetReassertSkipActive = "skip-active"
-	// NetReassertSkipNone: neither a real mount nor an armed trigger at the path — a removed or
-	// orphan state owned by the add/remove flows, not this reconcile.
-	NetReassertSkipNone = "skip-none"
+	// NetReassertSkipForeign: a non-network, non-autofs filesystem occupies the path (ext4/tmpfs/…) —
+	// not a state this reconcile owns, and re-arming over it would fail (mountpoint busy).
+	NetReassertSkipForeign = "skip-foreign"
 )
 
 // NetReassertResult is one share's outcome in a reassert pass.
@@ -39,24 +53,36 @@ type NetReassertResult struct {
 	Err    error  // set when the rearm action failed (skip rows never error)
 }
 
-// netReassertAction is the pure §8 decision: the /proc/mounts fstype at the share's mountpoint
-// ("" = nothing mounted there) → the action to take.
-func netReassertAction(fstype string) string {
+// Remediates reports whether an action expects the share to become visible in running guests (drives
+// the caller's guest-visibility verify). Skip-active also expects visibility (an inherited live mount);
+// only foreign-fs and errored rows expect nothing.
+func (r NetReassertResult) Remediates() bool {
+	return r.Err == nil && r.Action != NetReassertSkipForeign
+}
+
+// netReassertActive reports whether the fstype at a share's mountpoint is a live network mount — the
+// ONLY input to the skip-active decision (F11: never the automount unit's state). "" (a failed/disarmed
+// automount leaves NO /proc/mounts entry) and "autofs" (an armed-but-idle trigger) are BOTH not-active
+// and therefore re-arm targets; a foreign local fs is left alone.
+func netReassertClassify(fstype string) string {
 	switch {
 	case isNetworkMounted(fstype):
 		return NetReassertSkipActive
-	case fstype == "autofs":
+	case fstype == "" || fstype == "autofs":
+		// Not actively mounted, but a marker unit exists for this path: idle-armed, disarmed, OR
+		// failed/start-limited — all of them must be re-armed so a fresh trigger event propagates.
 		return NetReassertRearmed
 	default:
-		return NetReassertSkipNone
+		return NetReassertSkipForeign // ext4/tmpfs/… — foreign, not ours to churn
 	}
 }
 
 // ReassertNetworkAutomounts runs the reassert pass over every configured network mount: for each
-// installed pair, decide per netReassertAction and re-arm idle triggers. Returns one result per
-// share so callers (agent startup / guest-hook post-start) can verify guest visibility. Errors on
-// one share never stop the pass. Callers MUST NOT invoke this from periodic health paths — an idle
-// trigger is healthy, and the pass is only needed after a guest (re)start or at agent startup.
+// installed pair, decide per netReassertClassify and re-arm every not-active trigger (reset-failed first
+// if the unit is stuck). Returns one result per share so callers (agent startup / guest-hook post-start)
+// can verify guest visibility. Errors on one share never stop the pass. Callers MUST NOT invoke this
+// from periodic health paths — an idle trigger is healthy, and the pass is only needed after a guest
+// (re)start or at agent startup.
 func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReassertResult {
 	entries, err := h.networkUnitEntries()
 	if err != nil {
@@ -75,21 +101,27 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	}
 	var out []NetReassertResult
 	for _, e := range entries {
-		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertAction(fstypes[e.where])}
+		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertClassify(fstypes[e.where])}
 		switch res.Action {
 		case NetReassertSkipActive:
-			h.logger.Debug("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
-				"name", e.name, "where", e.where)
-		case NetReassertSkipNone:
-			h.logger.Debug("netreassert: no mount and no armed trigger — skip (removed/orphan state owned elsewhere)",
-				"name", e.name, "where", e.where)
+			h.logger.Info("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
+		case NetReassertSkipForeign:
+			h.logger.Info("netreassert: foreign filesystem at mountpoint — skip (not a network state we own)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
 		case NetReassertRearmed:
+			// F10: clear a failed/start-limit lockout FIRST or the enable --now is refused; the verdict
+			// records whether a reset was actually needed.
+			if h.resetNetworkAutomountIfFailed(ctx, e.where) {
+				res.Action = NetReassertResetRearmed
+			}
 			if err := h.rearmNetworkAutomount(ctx, e.where); err != nil {
 				res.Err = err
-				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where, "err", err)
+				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where,
+					"verdict", "error", "err", err)
 			} else {
 				h.logger.Info("netreassert: automount trigger re-armed (fresh mount event propagates into running guests)",
-					"name", e.name, "where", e.where)
+					"name", e.name, "where", e.where, "verdict", res.Action)
 			}
 		}
 		out = append(out, res)
@@ -97,6 +129,33 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	return out
 }
 
+// resetNetworkAutomountIfFailed clears a failed/start-limit-hit lockout on the share's unit pair so the
+// subsequent `enable --now` is not refused (F10 — the campaign's start-limited automount that no
+// platform path re-armed). Returns true when either unit was in the failed state (so the caller can
+// report the reset-failed+rearmed verdict). The failed-state read is unprivileged (`systemctl
+// is-failed`, seam-injected); the reset-failed is the new sudoers verb.
+func (h *SudoHostOps) resetNetworkAutomountIfFailed(ctx context.Context, where string) bool {
+	mountUnit, err := UnitNameForMount(where)
+	if err != nil {
+		return false
+	}
+	automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount"
+	reset := false
+	for _, unit := range []string{automountUnit, mountUnit} {
+		if !h.unitFailed(ctx, unit) {
+			continue
+		}
+		reset = true
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			// Tolerated: a reset-failed that itself fails still lets the enable --now try; log it.
+			h.logger.Warn("netreassert: reset-failed tolerated failure", "unit", unit, "err", err)
+		} else {
+			h.logger.Warn("netreassert: cleared failed/start-limit lockout before re-arm (F10)", "unit", unit)
+		}
+	}
+	return reset
+}
+
 // rearmNetworkAutomount stops then re-enables+starts the .automount for a mountpoint. The stop is
 // tolerated failing (unit not loaded); the enable --now is the action that must succeed. Both verbs
 // are the existing FELHOM_NETMOUNT sudoers grants.

internal/storage/netreassert_test.go

@@ -9,23 +9,23 @@ import (
 	"testing"
 )
 
-// The §8 decision table, encoded exactly (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1).
-func TestNetReassertAction_Table(t *testing.T) {
+// The §8 decision table, encoded exactly (CAMPAIGN-3 F11: fstype-driven, automount state IGNORED).
+func TestNetReassertClassify_Table(t *testing.T) {
 	cases := []struct {
 		fstype string
 		want   string
 	}{
-		{"nfs4", NetReassertSkipActive},  // real mount — inherited by fresh namespaces
-		{"nfs", NetReassertSkipActive},   // real mount
-		{"cifs", NetReassertSkipActive},  // real mount
-		{"autofs", NetReassertRearmed},   // idle trigger — NOT inherited, re-arm to propagate
-		{"", NetReassertSkipNone},        // nothing at the path — removed/orphan, owned elsewhere
-		{"ext4", NetReassertSkipNone},    // a local fs at the path is not a network state we own
-		{"tmpfs", NetReassertSkipNone},   //
+		{"nfs4", NetReassertSkipActive},   // real mount — inherited by fresh namespaces
+		{"nfs", NetReassertSkipActive},    // real mount
+		{"cifs", NetReassertSkipActive},   // real mount
+		{"autofs", NetReassertRearmed},    // idle trigger — NOT inherited, re-arm to propagate
+		{"", NetReassertRearmed},          // F10: a failed/disarmed automount leaves NO mount entry — re-arm
+		{"ext4", NetReassertSkipForeign},  // a foreign local fs at the path is not ours to churn
+		{"tmpfs", NetReassertSkipForeign}, //
 	}
 	for _, c := range cases {
-		if got := netReassertAction(c.fstype); got != c.want {
-			t.Errorf("netReassertAction(%q) = %q, want %q", c.fstype, got, c.want)
+		if got := netReassertClassify(c.fstype); got != c.want {
+			t.Errorf("netReassertClassify(%q) = %q, want %q", c.fstype, got, c.want)
 		}
 	}
 }
@@ -45,18 +45,26 @@ func installNetUnitFile(t *testing.T, unitDir string, spec NetworkMountSpec) (wh
 	return where, strings.TrimSuffix(mountUnit, ".mount") + ".automount"
 }
 
-func netReassertOps(t *testing.T, unitDir string, mounts []Mount) (*SudoHostOps, *recordingRunner) {
+// netReassertOps builds ops with a hermetic unitFailed seam (default: nothing failed — no shelling out
+// to a real systemctl is-failed). failedUnits, if set, marks specific unit names as failed.
+func netReassertOps(t *testing.T, unitDir string, mounts []Mount, failedUnits ...string) (*SudoHostOps, *recordingRunner) {
 	t.Helper()
 	rr := &recordingRunner{}
 	ops := NewSudoHostOps(SudoHostOpsConfig{
 		Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(),
 		Host: &fakeHostReader{mounts: mounts}, Logger: quietLogger(),
 	})
+	failed := map[string]bool{}
+	for _, u := range failedUnits {
+		failed[u] = true
+	}
+	ops.unitFailed = func(_ context.Context, unit string) bool { return failed[unit] }
 	return ops, rr
 }
 
 // An idle trigger (autofs at the mountpoint) must be re-armed with EXACTLY the granted verbs:
-// `systemctl stop -- <unit>.automount` then `systemctl enable --now -- <unit>.automount`.
+// `systemctl stop -- <unit>.automount` then `systemctl enable --now -- <unit>.automount`. No
+// reset-failed when nothing is failed.
 func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -70,11 +78,8 @@ func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	if len(results) != 1 || results[0].Action != NetReassertRearmed || results[0].Err != nil {
 		t.Fatalf("want one rearmed result, got %+v", results)
 	}
-	if results[0].Where != where || results[0].Name != "media" {
-		t.Fatalf("result identity wrong: %+v", results[0])
-	}
 	if len(rr.calls) != 2 {
-		t.Fatalf("want exactly stop + enable --now, got %d calls: %v", len(rr.calls), rr.calls)
+		t.Fatalf("want exactly stop + enable --now (no reset-failed when clean), got %d calls: %v", len(rr.calls), rr.calls)
 	}
 	stop, enable := strings.Join(rr.calls[0], " "), strings.Join(rr.calls[1], " ")
 	if !strings.Contains(stop, "systemctl stop -- "+autoUnit) {
@@ -85,8 +90,44 @@ func TestReassertNetworkAutomounts_RearmsIdleTrigger(t *testing.T) {
 	}
 }
 
+// F10: a failed/start-limited automount (the campaign's unexport→idle-timeout→access×5 sequence leaves
+// NO mount entry, fstype "") must be reset-failed FIRST, then re-armed — verdict reset-failed+rearmed.
+// Companion to the campaign's reboots #2–#4: the pre-0.85 code returned skip-none here and left it dead.
+func TestReassertNetworkAutomounts_ResetsFailedThenRearms(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where, autoUnit := installNetUnitFile(t, unitDir, spec)
+	// Nothing mounted (fstype "" — a start-limited automount), and the automount unit is in failed state.
+	ops, rr := netReassertOps(t, unitDir, nil, autoUnit)
+
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != 1 || results[0].Action != NetReassertResetRearmed || results[0].Err != nil {
+		t.Fatalf("want one reset-failed+rearmed result, got %+v", results)
+	}
+	// reset-failed <automount>, then stop + enable --now.
+	var sawReset, sawEnable bool
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		if strings.Contains(j, "reset-failed -- "+autoUnit) {
+			sawReset = true
+		}
+		if strings.Contains(j, "enable --now -- "+autoUnit) {
+			sawEnable = true
+		}
+	}
+	if !sawReset {
+		t.Errorf("a failed unit must be reset-failed before re-arm (F10); calls: %v", rr.calls)
+	}
+	if !sawEnable {
+		t.Errorf("the trigger must still be re-armed after reset-failed; calls: %v", rr.calls)
+	}
+}
+
 // An ACTIVE real mount must not be touched — stopping the automount of a live mount would churn it.
-// (Red-proof companion: a naive always-rearm implementation fails this with 2 recorded calls.)
+// (Red-proof companion: a naive always-rearm implementation fails this with recorded calls.)
 func TestReassertNetworkAutomounts_ActiveMountUntouched(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -105,27 +146,81 @@ func TestReassertNetworkAutomounts_ActiveMountUntouched(t *testing.T) {
 	}
 }
 
-// Neither a mount nor an armed trigger → skip (removed/orphan state, owned by add/remove flows).
-func TestReassertNetworkAutomounts_NoTriggerSkips(t *testing.T) {
+// The automount unit's own state is IGNORED (F11 red-proof): even though a mutant that consulted
+// `systemctl is-active <automount>` would see an armed trigger as "active" and skip it, the fstype at
+// the path is autofs (idle) so the correct code RE-ARMS. Encoded as: an idle trigger re-arms regardless
+// of failed/armed unit state — the decision is fstype only.
+func TestReassertNetworkAutomounts_IgnoresAutomountUnitState(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
 	}
 	unitDir := t.TempDir()
 	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
-	installNetUnitFile(t, unitDir, spec)
-	ops, rr := netReassertOps(t, unitDir, nil) // nothing at the mountpoint
-
+	where, _ := installNetUnitFile(t, unitDir, spec)
+	// fstype autofs (idle-armed): the correct decision is re-arm, NOT skip — a state-of-the-automount
+	// check would mis-skip an armed trigger (which always reports "active").
+	ops, _ := netReassertOps(t, unitDir, []Mount{{MountPoint: where, FSType: "autofs"}})
 	results := ops.ReassertNetworkAutomounts(context.Background())
-	if len(results) != 1 || results[0].Action != NetReassertSkipNone {
-		t.Fatalf("want one skip-none result, got %+v", results)
-	}
-	if len(rr.calls) != 0 {
-		t.Fatalf("a unit with no trigger must not be acted on, got: %v", rr.calls)
+	if len(results) != 1 || (results[0].Action != NetReassertRearmed && results[0].Action != NetReassertResetRearmed) {
+		t.Fatalf("an idle (autofs) trigger must re-arm regardless of automount unit state, got %+v", results)
 	}
 }
 
-// Idempotency: two consecutive passes over an idle trigger both succeed with the same action and no
-// error (re-arming a fresh trigger is harmless — same end state).
+// A foreign filesystem at the path (ext4/tmpfs) is skipped — re-arming over it would fail (busy).
+func TestReassertNetworkAutomounts_ForeignFSSkipped(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where, _ := installNetUnitFile(t, unitDir, spec)
+	ops, rr := netReassertOps(t, unitDir, []Mount{{MountPoint: where, FSType: "ext4"}})
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != 1 || results[0].Action != NetReassertSkipForeign {
+		t.Fatalf("want one skip-foreign result, got %+v", results)
+	}
+	if len(rr.calls) != 0 {
+		t.Fatalf("a foreign fs at the path must trigger zero systemctl calls, got: %v", rr.calls)
+	}
+}
+
+// F9: the pass returns exactly one verdict per installed unit — including a failed one. An
+// empty-looking sweep over N shares is structurally impossible (the campaign's silent zero-line sweep).
+func TestReassertNetworkAutomounts_VerdictPerUnit(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	specs := []NetworkMountSpec{
+		{Name: "alpha", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/a", MappedUID: 1000, MappedGID: 1000},
+		{Name: "beta", Protocol: ProtocolNFS, Server: "10.0.0.6", Export: "/srv/b", MappedUID: 1000, MappedGID: 1000},
+		{Name: "gamma", Protocol: ProtocolNFS, Server: "10.0.0.7", Export: "/srv/c", MappedUID: 1000, MappedGID: 1000},
+	}
+	for _, s := range specs {
+		installNetUnitFile(t, unitDir, s)
+	}
+	// alpha active (skip-active), beta idle (rearm), gamma failed-and-unmounted (reset-failed+rearm).
+	gammaAuto := func() string {
+		u, _ := UnitNameForMount(specs[2].Where())
+		return strings.TrimSuffix(u, ".mount") + ".automount"
+	}()
+	ops, _ := netReassertOps(t, unitDir, []Mount{
+		{MountPoint: specs[0].Where(), FSType: "nfs4"},
+		{MountPoint: specs[1].Where(), FSType: "autofs"},
+	}, gammaAuto)
+
+	results := ops.ReassertNetworkAutomounts(context.Background())
+	if len(results) != len(specs) {
+		t.Fatalf("verdict count must equal unit count (%d), got %d: %+v", len(specs), len(results), results)
+	}
+	for _, r := range results {
+		if r.Action == "" {
+			t.Errorf("every share must carry a verdict (F9), got empty for %s", r.Name)
+		}
+	}
+}
+
+// Idempotency: two consecutive passes over an idle trigger both re-arm with the same action, no error.
 func TestReassertNetworkAutomounts_Idempotent(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
@@ -140,9 +235,6 @@ func TestReassertNetworkAutomounts_Idempotent(t *testing.T) {
 	if first[0].Action != NetReassertRearmed || second[0].Action != NetReassertRearmed {
 		t.Fatalf("both passes must re-arm: first=%+v second=%+v", first, second)
 	}
-	if first[0].Err != nil || second[0].Err != nil {
-		t.Fatalf("idempotent passes must not error: first=%v second=%v", first[0].Err, second[0].Err)
-	}
 	if len(rr.calls) != 4 {
 		t.Fatalf("two passes = 2×(stop+enable), got %d: %v", len(rr.calls), rr.calls)
 	}

internal/storage/netresidue_test.go

@@ -0,0 +1,96 @@
+package storage
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// F2/F1 (CAMPAIGN-3): a remove (and, via the same path, a rolled-back add) must leave ZERO residue —
+// no failed-state units, no leftover mountpoint dir. RemoveNetworkMount must reset-failed the stuck
+// unit BEFORE removing the files (or systemd keeps it as not-found/failed) and rmdir the mountpoint.
+func TestRemoveNetworkMount_ZeroResidue(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	stageDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	where := spec.Where()
+	mountName, err := UnitNameForMount(where)
+	if err != nil {
+		t.Fatalf("unit name: %v", err)
+	}
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	// Both unit files present on disk (rm is recorded, so they stay for the assertion).
+	if err := os.WriteFile(filepath.Join(unitDir, mountName), []byte(renderNetworkMountUnit(spec)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(unitDir, autoName), []byte(renderNetworkAutomountUnit(spec)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: stageDir, Host: &fakeHostReader{}, Logger: quietLogger()})
+	// The automount is in the failed state (start-limit residue) — must be reset-failed.
+	ops.unitFailed = func(_ context.Context, unit string) bool { return unit == autoName }
+
+	if err := ops.RemoveNetworkMount(context.Background(), "media"); err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+
+	var sawReset, sawRmUnit, sawRmdir, sawReload bool
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		switch {
+		case strings.Contains(j, "reset-failed -- "+autoName):
+			sawReset = true
+		case strings.Contains(j, "rm -f") && strings.Contains(j, autoName):
+			sawRmUnit = true
+		case strings.Contains(j, "rmdir") && strings.Contains(j, where):
+			sawRmdir = true
+		case strings.Contains(j, "daemon-reload"):
+			sawReload = true
+		}
+	}
+	if !sawReset {
+		t.Errorf("F2: a failed unit must be reset-failed on remove; calls: %v", rr.calls)
+	}
+	if !sawRmUnit {
+		t.Errorf("the unit files must be removed; calls: %v", rr.calls)
+	}
+	if !sawRmdir {
+		t.Errorf("F1: the empty mountpoint dir must be rmdir'd; calls: %v", rr.calls)
+	}
+	if !sawReload {
+		t.Errorf("daemon-reload must run after removal; calls: %v", rr.calls)
+	}
+}
+
+// rmdir is used (never rm -rf) — the fail-safe: a non-empty dir is left in place, not force-removed.
+func TestRemoveNetworkMount_NeverForceRemovesMountpoint(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("systemd-escaped unit filename contains a backslash; exercised on the Linux build server")
+	}
+	unitDir := t.TempDir()
+	spec := NetworkMountSpec{Name: "media", Protocol: ProtocolNFS, Server: "10.0.0.5", Export: "/srv/media", MappedUID: 1000, MappedGID: 1000}
+	mountName, _ := UnitNameForMount(spec.Where())
+	autoName := strings.TrimSuffix(mountName, ".mount") + ".automount"
+	_ = os.WriteFile(filepath.Join(unitDir, mountName), []byte(renderNetworkMountUnit(spec)), 0o644)
+	_ = os.WriteFile(filepath.Join(unitDir, autoName), []byte(renderNetworkAutomountUnit(spec)), 0o644)
+
+	rr := &recordingRunner{}
+	ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
+	if err := ops.RemoveNetworkMount(context.Background(), "media"); err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+	for _, c := range rr.calls {
+		j := strings.Join(c, " ")
+		if strings.Contains(j, "rm -rf") || (strings.Contains(j, "rm ") && strings.Contains(j, "/mnt/felhom-drives/media") && !strings.Contains(j, "rmdir")) {
+			t.Errorf("mountpoint cleanup must be rmdir-only (never rm -rf); offending call: %s", j)
+		}
+	}
+}