felhom-agent/internal/storage/netreassert.go

package storage

import (
	"context"
	"fmt"
	"strings"
)

// Network-mount guest-reboot / boot reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1, hardened by
// CAMPAIGN-3 F9/F10/F11).
//
// THE BUG IT FIXES: a fresh guest namespace inherits REAL submounts of the shared parent (ext4, or an
// actively-mounted nfs4/cifs) but NOT an idle autofs trigger — so after any guest reboot an idle NAS
// share silently degrades to a local stub directory inside the guest. The heal is host-side and
// host-global: re-creating the .automount unit emits a FRESH trigger-mount event, which propagates live
// into every running guest's slave bind (live-proven in the RCA remediation, 2026-07-11).
//
// CAMPAIGN-3 hardening:
//   - F11 (read the right unit): the decision is driven ONLY by the host `/proc/mounts` fstype AT the
//     mountpoint — an ACTIVE real mount (nfs4/cifs) is inherited and left alone; anything else is
//     re-armed. The `.automount` unit's own state is NEVER consulted (an armed trigger always reports
//     "active", which is exactly why a state-of-the-automount check mis-skips idle triggers).
//   - F10 (re-arm for real): a `.mount`/`.automount` left in `failed`/start-limit-hit state (the
//     campaign's unexport→idle-timeout→access×5 sequence) is `reset-failed` FIRST — without it the
//     `enable --now` below is refused by the start limit and the share stays dead across every boot.
//   - F9 (say what you did): the pass enumerates by the marker-owned unit files on disk (not by
//     enablement or runtime state) and logs an INFO verdict line for EVERY share — an empty-looking
//     sweep over N shares is structurally impossible.
//
// The action uses the sudoers-granted verbs only (`systemctl reset-failed -- *`, `systemctl stop -- *`,
// `systemctl enable --now -- *`). Idempotent: re-arming an already-armed trigger recreates it — same end
// state, and the fresh mount event is harmless. An ACTIVE real mount is never touched.

// Reassert actions (the §8 decision table, encoded — CAMPAIGN-3 F11).
const (
	// NetReassertRearmed: the trigger was re-created (stop + enable --now) — the propagation heal.
	NetReassertRearmed = "rearmed"
	// NetReassertResetRearmed: the unit was failed/start-limited, reset-failed, THEN re-armed (F10).
	NetReassertResetRearmed = "reset-failed+rearmed"
	// NetReassertSkipActive: a real network fs is mounted at the path — inherited by fresh namespaces,
	// nothing to do (verify only).
	NetReassertSkipActive = "skip-active"
	// NetReassertSkipForeign: a non-network, non-autofs filesystem occupies the path (ext4/tmpfs/…) —
	// not a state this reconcile owns, and re-arming over it would fail (mountpoint busy).
	NetReassertSkipForeign = "skip-foreign"
)

// NetReassertResult is one share's outcome in a reassert pass.
type NetReassertResult struct {
	Name   string // share name (mount dir basename)
	Where  string // guest-visible mountpoint (/mnt/felhom-drives/<name>)
	Action string // NetReassert* constant
	Err    error  // set when the rearm action failed (skip rows never error)
}

// Remediates reports whether an action expects the share to become visible in running guests (drives
// the caller's guest-visibility verify). Skip-active also expects visibility (an inherited live mount);
// only foreign-fs and errored rows expect nothing.
func (r NetReassertResult) Remediates() bool {
	return r.Err == nil && r.Action != NetReassertSkipForeign
}

// netReassertActive reports whether the fstype at a share's mountpoint is a live network mount — the
// ONLY input to the skip-active decision (F11: never the automount unit's state). "" (a failed/disarmed
// automount leaves NO /proc/mounts entry) and "autofs" (an armed-but-idle trigger) are BOTH not-active
// and therefore re-arm targets; a foreign local fs is left alone.
func netReassertClassify(fstype string) string {
	switch {
	case isNetworkMounted(fstype):
		return NetReassertSkipActive
	case fstype == "" || fstype == "autofs":
		// Not actively mounted, but a marker unit exists for this path: idle-armed, disarmed, OR
		// failed/start-limited — all of them must be re-armed so a fresh trigger event propagates.
		return NetReassertRearmed
	default:
		return NetReassertSkipForeign // ext4/tmpfs/… — foreign, not ours to churn
	}
}

// ReassertNetworkAutomounts runs the reassert pass over every configured network mount: for each
// installed pair, decide per netReassertClassify and re-arm every not-active trigger (reset-failed first
// if the unit is stuck). Returns one result per share so callers (agent startup / guest-hook post-start)
// can verify guest visibility. Errors on one share never stop the pass. Callers MUST NOT invoke this
// from periodic health paths — an idle trigger is healthy, and the pass is only needed after a guest
// (re)start or at agent startup.
func (h *SudoHostOps) ReassertNetworkAutomounts(ctx context.Context) []NetReassertResult {
	entries, err := h.networkUnitEntries()
	if err != nil {
		h.logger.Warn("netreassert: unit enumeration failed", "err", err)
		return nil
	}
	fstypes := map[string]string{}
	if h.host != nil {
		if mounts, merr := h.host.Mounts(); merr == nil {
			for _, m := range mounts {
				fstypes[m.MountPoint] = m.FSType
			}
		} else {
			h.logger.Warn("netreassert: mount-table read failed — treating all shares as unmounted", "err", merr)
		}
	}
	var out []NetReassertResult
	for _, e := range entries {
		res := NetReassertResult{Name: e.name, Where: e.where, Action: netReassertClassify(fstypes[e.where])}
		switch res.Action {
		case NetReassertSkipActive:
			h.logger.Info("netreassert: share actively mounted — skip (fresh namespaces inherit real mounts)",
				"name", e.name, "where", e.where, "verdict", res.Action)
		case NetReassertSkipForeign:
			h.logger.Info("netreassert: foreign filesystem at mountpoint — skip (not a network state we own)",
				"name", e.name, "where", e.where, "verdict", res.Action)
		case NetReassertRearmed:
			// F10: clear a failed/start-limit lockout FIRST or the enable --now is refused; the verdict
			// records whether a reset was actually needed.
			if h.resetNetworkAutomountIfFailed(ctx, e.where) {
				res.Action = NetReassertResetRearmed
			}
			if err := h.rearmNetworkAutomount(ctx, e.where); err != nil {
				res.Err = err
				h.logger.Warn("netreassert: trigger re-arm failed", "name", e.name, "where", e.where,
					"verdict", "error", "err", err)
			} else {
				h.logger.Info("netreassert: automount trigger re-armed (fresh mount event propagates into running guests)",
					"name", e.name, "where", e.where, "verdict", res.Action)
			}
		}
		out = append(out, res)
	}
	return out
}

// resetNetworkAutomountIfFailed clears a failed/start-limit-hit lockout on the share's unit pair so the
// subsequent `enable --now` is not refused (F10 — the campaign's start-limited automount that no
// platform path re-armed). Returns true when either unit was in the failed state (so the caller can
// report the reset-failed+rearmed verdict). The failed-state read is unprivileged (`systemctl
// is-failed`, seam-injected); the reset-failed is the new sudoers verb.
func (h *SudoHostOps) resetNetworkAutomountIfFailed(ctx context.Context, where string) bool {
	mountUnit, err := UnitNameForMount(where)
	if err != nil {
		return false
	}
	automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount"
	reset := false
	for _, unit := range []string{automountUnit, mountUnit} {
		if !h.unitFailed(ctx, unit) {
			continue
		}
		reset = true
		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
			// Tolerated: a reset-failed that itself fails still lets the enable --now try; log it.
			h.logger.Warn("netreassert: reset-failed tolerated failure", "unit", unit, "err", err)
		} else {
			h.logger.Warn("netreassert: cleared failed/start-limit lockout before re-arm (F10)", "unit", unit)
		}
	}
	return reset
}

// RearmNetworkAutomount is the caller-driven remediation (CAMPAIGN-3 F11, matrix-corrected): when a
// running guest does NOT see a share after the reassert pass, re-arm its trigger regardless of the
// host's mount state. LIVE FINDING (matrix row 1/2, 2026-07-12): a freshly-rebooted guest does NOT
// inherit an ACTIVE host mount either — only a fresh trigger-mount event propagates into the new
// namespace — so skip-active is safe to skip PROACTIVELY (don't churn shares guests already see) but
// must be remediated REACTIVELY when a specific guest is blind. reset-failed first (F10), then re-arm.
func (h *SudoHostOps) RearmNetworkAutomount(ctx context.Context, where string) error {
	h.resetNetworkAutomountIfFailed(ctx, where)
	return h.rearmNetworkAutomount(ctx, where)
}

// rearmNetworkAutomount stops then re-enables+starts the .automount for a mountpoint. The stop is
// tolerated failing (unit not loaded); the enable --now is the action that must succeed. Both verbs
// are the existing FELHOM_NETMOUNT sudoers grants.
func (h *SudoHostOps) rearmNetworkAutomount(ctx context.Context, where string) error {
	mountUnit, err := UnitNameForMount(where)
	if err != nil {
		return fmt.Errorf("netreassert: unit name for %s: %w", where, err)
	}
	automountUnit := strings.TrimSuffix(mountUnit, ".mount") + ".automount"
	if err := h.run(ctx, h.bins.Systemctl, "stop", "--", automountUnit); err != nil {
		// Tolerated: a not-loaded unit still enables cleanly below.
		h.logger.Debug("netreassert: automount stop tolerated failure", "unit", automountUnit, "err", err)
	}
	if err := h.run(ctx, h.bins.Systemctl, "enable", "--now", "--", automountUnit); err != nil {
		return fmt.Errorf("netreassert: enabling automount %s: %w", automountUnit, err)
	}
	return nil
}