agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/storage/netreassert.go

@@ -6,29 +6,43 @@ import (
 	"strings"
 )
 
-// Network-mount guest-reboot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1).
+// Network-mount guest-reboot / boot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1, hardened by
+// CAMPAIGN-3 F9/F10/F11).
 //
-// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or
-// an actively-mounted nfs4) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
+// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or an
+// actively-mounted nfs4/cifs) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
 // share silently degrades to a local stub directory inside the guest. The heal is host-side and
-// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates
-// live into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
+// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates live
+// into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
 //
-// The action uses the sudoers-granted verbs only (`systemctl stop -- *.automount` +
-// `systemctl enable --now -- *.automount`; there is NO restart grant). Idempotent: re-arming an
-// already-armed trigger just recreates it — same end state, and the fresh mount event is harmless.
-// An ACTIVE real mount is never touched (stopping the automount of a live mount would churn it).
+// CAMPAIGN-3 hardening:
+//   - F11 (read the right unit): the decision is driven ONLY by the host `/proc/mounts` fstype AT the
+//     mountpoint — an ACTIVE real mount (nfs4/cifs) is inherited and left alone; anything else is
+//     re-armed. The `.automount` unit's own state is NEVER consulted (an armed trigger always reports
+//     "active", which is exactly why a state-of-the-automount check mis-skips idle triggers).
+//   - F10 (re-arm for real): a `.mount`/`.automount` left in `failed`/start-limit-hit state (the
+//     campaign's unexport→idle-timeout→access×5 sequence) is `reset-failed` FIRST — without it the
+//     `enable --now` below is refused by the start limit and the share stays dead across every boot.
+//   - F9 (say what you did): the pass enumerates by the marker-owned unit files on disk (not by
+//     enablement or runtime state) and logs an INFO verdict line for EVERY share — an empty-looking
+//     sweep over N shares is structurally impossible.
+//
+// The action uses the sudoers-granted verbs only (`systemctl reset-failed -- *`, `systemctl stop -- *`,
+// `systemctl enable --now -- *`). Idempotent: re-arming an already-armed trigger recreates it — same end
+// state, and the fresh mount event is harmless. An ACTIVE real mount is never touched.
 
-// Reassert actions (the §8 decision table, encoded).
+// Reassert actions (the §8 decision table, encoded — CAMPAIGN-3 F11).
 const (
 	// NetReassertRearmed: the trigger was re-created (stop + enable --now) — the propagation heal.
 	NetReassertRearmed = "rearmed"
-	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh
-	// namespaces, nothing to do (verify only).
+	// NetReassertResetRearmed: the unit was failed/start-limited, reset-failed, THEN re-armed (F10).
+	NetReassertResetRearmed = "reset-failed+rearmed"
+	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh namespaces,
+	// nothing to do (verify only).
 	NetReassertSkipActive = "skip-active"
-	// NetReassertSkipNone: neither a real mount nor an armed trigger at the path — a removed or
-	// orphan state owned by the add/remove flows, not this reconcile.
-	NetReassertSkipNone = "skip-none"
+	// NetReassertSkipForeign: a non-network, non-autofs filesystem occupies the path (ext4/tmpfs/…) —
+	// not a state this reconcile owns, and re-arming over it would fail (mountpoint busy).
+	NetReassertSkipForeign = "skip-foreign"
 )
 
 // NetReassertResult is one share's outcome in a reassert pass.
@@ -39,24 +53,36 @@ type NetReassertResult struct {
 	Err    error  // set when the rearm action failed (skip rows never error)
 }
 
-// netReassertAction is the pure §8 decision: the /proc/mounts fstype at the share's mountpoint
-// ("" = nothing mounted there) → the action to take.
-func netReassertAction(fstype string) string {
+// Remediates reports whether an action expects the share to become visible in running guests (drives
+// the caller's guest-visibility verify). Skip-active also expects visibility (an inherited live mount);
+// only foreign-fs and errored rows expect nothing.
+func (r NetReassertResult) Remediates() bool {
+	return r.Err == nil && r.Action != NetReassertSkipForeign
+}
+
+// netReassertActive reports whether the fstype at a share's mountpoint is a live network mount — the
+// ONLY input to the skip-active decision (F11: never the automount unit's state). "" (a failed/disarmed
+// automount leaves NO /proc/mounts entry) and "autofs" (an armed-but-idle trigger) are BOTH not-active
+// and therefore re-arm targets; a foreign local fs is left alone.
+func netReassertClassify(fstype string) string {
 	switch {
 	case isNetworkMounted(fstype):
 		return NetReassertSkipActive
-	case fstype == "autofs":
+	case fstype == "" || fstype == "autofs":
+		// Not actively mounted, but a marker unit exists for this path: idle-armed, disarmed, OR
+		// failed/start-limited — all of them must be re-armed so a fresh trigger event propagates.
 		return NetReassertRearmed
 	default:
-		return NetReassertSkipNone
+		return NetReassertSkipForeign // ext4/tmpfs/… — foreign, not ours to churn
 	}
 }
 
 // ReassertNetworkAutomounts runs the reassert pass over every configured network mount: for each
-// installed pair, decide per netReassertAction and re-arm idle triggers. Returns one result per
-// share so callers (agent startup / guest-hook post-start) can verify guest visibility. Errors on
-// one share never stop the pass. Callers MUST NOT invoke this from periodic health paths — an idle
-// trigger is healthy, and the pass is only needed after a guest (re)start or at agent startup.
+// installed pair, decide per netReassertClassify and re-arm every not-active trigger (reset-failed first
+// if the unit is stuck). Returns one result per share so callers (agent startup / guest-hook post-start)
+// can verify guest visibility. Errors on one share never stop the pass. Callers MUST NOT invoke this
+// from periodic health paths — an idle trigger is healthy, and the pass is only needed after a guest
+// (re)start or at agent startup.
 func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReassertResult {
 	entries, err := h.networkUnitEntries()
 	if err != nil {
@@ -75,21 +101,27 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	}
 	var out []NetReassertResult
 	for _, e := range entries {
-		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertAction(fstypes[e.where])}
+		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertClassify(fstypes[e.where])}
 		switch res.Action {
 		case NetReassertSkipActive:
-			h.logger.Debug("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
-				"name", e.name, "where", e.where)
-		case NetReassertSkipNone:
-			h.logger.Debug("netreassert: no mount and no armed trigger — skip (removed/orphan state owned elsewhere)",
-				"name", e.name, "where", e.where)
+			h.logger.Info("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
+		case NetReassertSkipForeign:
+			h.logger.Info("netreassert: foreign filesystem at mountpoint — skip (not a network state we own)",
+				"name", e.name, "where", e.where, "verdict", res.Action)
 		case NetReassertRearmed:
+			// F10: clear a failed/start-limit lockout FIRST or the enable --now is refused; the verdict
+			// records whether a reset was actually needed.
+			if h.resetNetworkAutomountIfFailed(ctx, e.where) {
+				res.Action = NetReassertResetRearmed
+			}
 			if err := h.rearmNetworkAutomount(ctx, e.where); err != nil {
 				res.Err = err
-				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where, "err", err)
+				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where,
+					"verdict", "error", "err", err)
 			} else {
 				h.logger.Info("netreassert: automount trigger re-armed (fresh mount event propagates into running guests)",
-					"name", e.name, "where", e.where)
+					"name", e.name, "where", e.where, "verdict", res.Action)
 			}
 		}
 		out = append(out, res)
@@ -97,6 +129,33 @@ func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReasse
 	return out
 }
 
+// resetNetworkAutomountIfFailed clears a failed/start-limit-hit lockout on the share's unit pair so the
+// subsequent `enable --now` is not refused (F10 — the campaign's start-limited automount that no
+// platform path re-armed). Returns true when either unit was in the failed state (so the caller can
+// report the reset-failed+rearmed verdict). The failed-state read is unprivileged (`systemctl
+// is-failed`, seam-injected); the reset-failed is the new sudoers verb.
+func (h *SudoHostOps) resetNetworkAutomountIfFailed(ctx context.Context, where string) bool {
+	mountUnit, err := UnitNameForMount(where)
+	if err != nil {
+		return false
+	}
+	automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount"
+	reset := false
+	for _, unit := range []string{automountUnit, mountUnit} {
+		if !h.unitFailed(ctx, unit) {
+			continue
+		}
+		reset = true
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			// Tolerated: a reset-failed that itself fails still lets the enable --now try; log it.
+			h.logger.Warn("netreassert: reset-failed tolerated failure", "unit", unit, "err", err)
+		} else {
+			h.logger.Warn("netreassert: cleared failed/start-limit lockout before re-arm (F10)", "unit", unit)
+		}
+	}
+	return reset
+}
+
 // rearmNetworkAutomount stops then re-enables+starts the .automount for a mountpoint. The stop is
 // tolerated failing (unit not loaded); the enable --now is the action that must succeed. Both verbs
 // are the existing FELHOM_NETMOUNT sudoers grants.